diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/pom.xml b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/pom.xml index 9061e7d6d0b..075e4c51b62 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/pom.xml +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/pom.xml @@ -160,6 +160,11 @@ legend-engine-xt-sql-pure ${project.version} + + org.finos.legend.engine + legend-engine-xt-relationalStore-postgresSqlModel-pure + ${project.version} + @@ -174,6 +179,10 @@ org.finos.legend.pure legend-pure-m3-core + + org.finos.legend.engine + legend-engine-xt-relationalStore-postgresSqlModel-pure + org.finos.legend.pure legend-pure-m2-dsl-graph-pure diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality.definition.json b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality.definition.json index f60163d36ef..b460570bf9d 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality.definition.json +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality.definition.json @@ -10,6 +10,7 @@ "core_functions_standard", "core_functions_standard", "core_external_query_sql", + "core_external_store_relational_postgres_sql_model", "platform", "platform_dsl_store", "platform_dsl_graph", diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/generation/dataqualitysqlextension.pure b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/generation/dataqualitysqlextension.pure new file mode 100644 index 00000000000..31beb0dd6f3 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/generation/dataqualitysqlextension.pure @@ -0,0 +1,48 @@ + +// Copyright 2026 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::external::query::sql::*; +import meta::external::query::sql::transformation::queryToPure::*; +import meta::external::query::sql::metamodel::*; +import meta::external::query::sql::transformation::utils::*; +import meta::external::query::sql::transformation::compile::utils::*; +import meta::external::dataquality::dataprofile::*; +import meta::pure::metamodel::relation::*; +import meta::external::query::sql::transformation::queryToPure::*; + +function <> meta::external::dataquality::dataProfileCoreExtension():SQLExtension[1] +{ + ^SQLExtension( + name = 'dq', + udtfs = ^UserDefinedTableFunctions( + prefix = 'core', + processors = [ + misc('data_profile', + meta::pure::metamodel::relation::Relation, + {args, fc, expCtx, ctx | + + assert($args->size() == 1, | 'expects single relational table'); + + let relationLambda = $args->at(0)->cast(@InstanceValue).values->cast(@LambdaFunction)->toOne(); + + let profilingLambda = iv(getProfilingLambda($relationLambda, [], [], false)); + + sfe(eval_Function_1__V_m_, $profilingLambda); + } + ) + ] + ) + ) +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/test/testTranspile.pure b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/test/testTranspile.pure new file mode 100644 index 00000000000..d64f0ce7fd6 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/test/testTranspile.pure @@ -0,0 +1,143 @@ +// Copyright 2026 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +###Pure +import meta::pure::functions::variant::navigation::*; +import meta::pure::functions::variant::convert::*; +import meta::pure::functions::hash::*; +import meta::relational::extension::*; +import meta::external::query::sql::transformation::queryToPure::*; +import meta::external::query::sql::metamodel::*; +import meta::external::query::sql::transformation::queryToPure::tests::*; +import meta::external::query::sql::*; +import meta::external::query::sql::schema::metamodel::*; +import meta::legend::service::metamodel::*; +import meta::pure::functions::meta::*; +import meta::pure::mapping::*; +import meta::pure::metamodel::serialization::grammar::*; +import meta::external::query::sql::transformation::compile::utils::*; + +//SELECT +function <>meta::external::dataquality::tests::testDataProfile():Boolean[1] +{ + test( + 'SELECT * FROM core_data_profile((SELECT * FROM service."/service/service1"))', + + [ + {| eval(|let cte = meta::external::query::sql::transformation::queryToPure::tests::FlatInput.all()->project( + ~[ Boolean: x | $x.booleanIn, Integer: x | $x.integerIn, Float: x | $x.floatIn, Decimal: x | $x.decimalIn, StrictDate: x | $x.strictDateIn, DateTime: x | $x.dateTimeIn, String: x | $x.stringIn] + ); + $cte->aggregate(~[ + column_name: x | 'Boolean': y | 'Boolean', + count: x | $x.Boolean : y | $y->count(), + count_distinct: x | $x.Boolean : y | $y->distinct()->count(), + count_null: x | if($x.Boolean->isEmpty(), |1, |[]) : y | $y->count(), + + 'number/max': x | []->cast(@Number) : y | $y->max(), + 'number/min': x | []->cast(@Number) : y | $y->min(), + 'date/max': x | []->cast(@Date) : y | $y->max(), + 'date/min': x | []->cast(@Date) : y | $y->min(), + 'string/max_length': x | []->cast(@Integer) : y | $y->max(), + 'string/min_length': x | []->cast(@Integer) : y | $y->min() + ]) + ->concatenate( + $cte->aggregate(~[ + column_name: x | 'Integer': y | 'Integer', + count: x | $x.Integer : y | $y->count(), + count_distinct: x | $x.Integer : y | $y->distinct()->count(), + count_null: x | if($x.Integer->isEmpty(), |1, |[]) : y | $y->count(), + + 'number/max': x | $x.Integer->cast(@Number) : y | $y->max(), + 'number/min': x | $x.Integer->cast(@Number) : y | $y->min(), + 'date/max': x | []->cast(@Date) : y | $y->max(), + 'date/min': x | []->cast(@Date) : y | $y->min(), + 'string/max_length': x | []->cast(@Integer) : y | $y->max(), + 'string/min_length': x | []->cast(@Integer) : y | $y->min() + ]) + )->concatenate( + $cte->aggregate(~[ + column_name: x | 'Float': y | 'Float', + count: x | $x.Float : y | $y->count(), + count_distinct: x | $x.Float : y | $y->distinct()->count(), + count_null: x | if($x.Float->isEmpty(), |1, |[]) : y | $y->count(), + + 'number/max': x | $x.Float->cast(@Number) : y | $y->max(), + 'number/min': x | $x.Float->cast(@Number) : y | $y->min(), + 'date/max': x | []->cast(@Date) : y | $y->max(), + 'date/min': x | []->cast(@Date) : y | $y->min(), + 'string/max_length': x | []->cast(@Integer) : y | $y->max(), + 'string/min_length': x | []->cast(@Integer) : y | $y->min() + ]) + )->concatenate( + $cte->aggregate(~[ + column_name: x | 'Decimal': y | 'Decimal', + count: x | $x.Decimal : y | $y->count(), + count_distinct: x | $x.Decimal : y | $y->distinct()->count(), + count_null: x | if($x.Decimal->isEmpty(), |1, |[]) : y | $y->count(), + + 'number/max': x | $x.Decimal->cast(@Number) : y | $y->max(), + 'number/min': x | $x.Decimal->cast(@Number) : y | $y->min(), + 'date/max': x | []->cast(@Date) : y | $y->max(), + 'date/min': x | []->cast(@Date) : y | $y->min(), + 'string/max_length': x | []->cast(@Integer) : y | $y->max(), + 'string/min_length': x | []->cast(@Integer) : y | $y->min() + ]) + )->concatenate( + $cte->aggregate(~[ + column_name: x | 'StrictDate': y | 'StrictDate', + count: x | $x.StrictDate : y | $y->count(), + count_distinct: x | $x.StrictDate : y | $y->distinct()->count(), + count_null: x | if($x.StrictDate->isEmpty(), |1, |[]) : y | $y->count(), + + 'number/max': x | []->cast(@Number) : y | $y->max(), + 'number/min': x | []->cast(@Number) : y | $y->min(), + 'date/max': x | $x.StrictDate->cast(@Date) : y | $y->max(), + 'date/min': x | $x.StrictDate->cast(@Date) : y | $y->min(), + 'string/max_length': x | []->cast(@Integer) : y | $y->max(), + 'string/min_length': x | []->cast(@Integer) : y | $y->min() + ]) + )->concatenate( + $cte->aggregate(~[ + column_name: x | 'DateTime': y | 'DateTime', + count: x | $x.DateTime : y | $y->count(), + count_distinct: x | $x.DateTime : y | $y->distinct()->count(), + count_null: x | if($x.DateTime->isEmpty(), |1, |[]) : y | $y->count(), + + 'number/max': x | []->cast(@Number) : y | $y->max(), + 'number/min': x | []->cast(@Number) : y | $y->min(), + 'date/max': x | $x.DateTime->cast(@Date) : y | $y->max(), + 'date/min': x | $x.DateTime->cast(@Date) : y | $y->min(), + 'string/max_length': x | []->cast(@Integer) : y | $y->max(), + 'string/min_length': x | []->cast(@Integer) : y | $y->min() + ]) + )->concatenate( + $cte->aggregate(~[ + column_name: x | 'String': y | 'String', + count: x | $x.String : y | $y->count(), + count_distinct: x | $x.String : y | $y->distinct()->count(), + count_null: x | if($x.String->isEmpty(), |1, |[]) : y | $y->count(), + + 'number/max': x | []->cast(@Number) : y | $y->max(), + 'number/min': x | []->cast(@Number) : y | $y->min(), + 'date/max': x | []->cast(@Date) : y | $y->max(), + 'date/min': x | []->cast(@Date) : y | $y->min(), + 'string/max_length': x | $x.String->length() : y | $y->max(), + 'string/min_length': x | $x.String->length() : y | $y->min() + ]) + ); + ) + } + ] + ) +} \ No newline at end of file diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-postgresSql/legend-engine-xt-relationalStore-postgresSql-grammar/src/main/java/org/finos/legend/engine/language/sql/grammar/from/SqlVisitor.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-postgresSql/legend-engine-xt-relationalStore-postgresSql-grammar/src/main/java/org/finos/legend/engine/language/sql/grammar/from/SqlVisitor.java index 9e2085c40dd..a8bb5237d7b 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-postgresSql/legend-engine-xt-relationalStore-postgresSql-grammar/src/main/java/org/finos/legend/engine/language/sql/grammar/from/SqlVisitor.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-postgresSql/legend-engine-xt-relationalStore-postgresSql-grammar/src/main/java/org/finos/legend/engine/language/sql/grammar/from/SqlVisitor.java @@ -1522,8 +1522,12 @@ public Node visitNestedExpression(SqlBaseParser.NestedExpressionContext context) @Override public Node visitSubqueryExpression(SqlBaseParser.SubqueryExpressionContext context) { -// return new SubqueryExpression((Query) visit(context.query())); - return unsupported("Subquery Expression"); + Query query = (Query) visit(context.queryStatement()); + + SubqueryExpression expr = new SubqueryExpression(); + expr.query = query; + + return expr; } @Override diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-postgresSql/legend-engine-xt-relationalStore-postgresSql-grammar/src/test/java/org/finos/legend/engine/language/sql/grammar/test/roundtrip/TestSQLRoundTrip.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-postgresSql/legend-engine-xt-relationalStore-postgresSql-grammar/src/test/java/org/finos/legend/engine/language/sql/grammar/test/roundtrip/TestSQLRoundTrip.java index ddd165a177e..3f11e2db84b 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-postgresSql/legend-engine-xt-relationalStore-postgresSql-grammar/src/test/java/org/finos/legend/engine/language/sql/grammar/test/roundtrip/TestSQLRoundTrip.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-postgresSql/legend-engine-xt-relationalStore-postgresSql-grammar/src/test/java/org/finos/legend/engine/language/sql/grammar/test/roundtrip/TestSQLRoundTrip.java @@ -369,6 +369,9 @@ public void testNested() check("SELECT * from (SELECT col from myTable)"); } + @Test + public void testTableSubQuery() {check("SELECT * from ((SELECT * from myTable))");} + @Test public void testCommonTableExpressionSingle() { diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/extension.pure b/legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/extension.pure index daf577f4ff5..35c52730b55 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/extension.pure +++ b/legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/extension.pure @@ -18,10 +18,15 @@ Class meta::external::query::sql::SQLExtension { name:String[1]; udfs: meta::external::query::sql::UserDefinedFunctions[0..1]; + udtfs: meta::external::query::sql::UserDefinedTableFunctions[0..1]; udf(name:String[1]){ $this.udfs.processors->filter(p | toLower($this.udfs->toOne().prefix + '_' + $p.name) == $name->toLower()) }:meta::external::query::sql::transformation::queryToPure::FunctionProcessor[*]; + + udtf(name:String[1]){ + $this.udtfs.processors->filter(p | toLower($this.udtfs->toOne().prefix + '_' + $p.name) == $name->toLower()) + }:meta::external::query::sql::transformation::queryToPure::FunctionProcessor[*]; } Class meta::external::query::sql::UserDefinedFunctions[ @@ -37,6 +42,19 @@ Class meta::external::query::sql::UserDefinedFunctions[ }:String[*]; } +Class meta::external::query::sql::UserDefinedTableFunctions[ + prefixNotBlank: $this.prefix->length() > 0, + prefixOnlyChars: $this.prefix->matches('[a-zA-Z]*') +] +{ + prefix: String[1]; + processors: meta::external::query::sql::transformation::queryToPure::FunctionProcessor[*]; + + names(){ + $this.processors->map(p | $this.prefix + '_' + $p.name) + }:String[*]; +} + Profile meta::external::query::sql::sql { stereotypes: [Extension]; diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/fromPure.pure b/legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/fromPure.pure index e6a695f6e02..77162a9cb91 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/fromPure.pure +++ b/legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/fromPure.pure @@ -32,6 +32,7 @@ import meta::pure::executionPlan::*; import meta::external::query::sql::transformation::utils::*; import meta::external::query::sql::transformation::compile::utils::*; import meta::pure::functions::relation::*; +import meta::pure::functions::meta::*; Class meta::external::query::sql::transformation::queryToPure::SQLSourceArgument { @@ -2098,6 +2099,7 @@ function <> meta::external::query::sql::transformation::queryToP s:SimpleCaseExpression[1] | processSimpleCaseExpression($s, $expContext, $context), s:SearchedCaseExpression[1] | processSearchedCaseExpression($s, $expContext, $context), s:SubscriptExpression[1] | processSubscriptExpression($s, $expContext, $context), + s:SubqueryExpression[1] | processSubqueryExpression($s.query, $context), t:Trim[1] | processTrim($t, $expContext, $context), e:meta::external::query::sql::metamodel::Expression[*] | fail('Expression type not yet supported'); iv(1); ])->evaluateAndDeactivate(); @@ -2556,6 +2558,15 @@ function <> meta::external::query::sql::transformation::queryToP sfe(at_T_MANY__Integer_1__T_1_, [$value, $index]); } +function <> meta::external::query::sql::transformation::queryToPure::processSubqueryExpression(query: Query[1], context:SqlTransformContext[1]):ValueSpecification[1] +{ + let subQueryCtx = processQuery($query, $context); + + let lambda = $subQueryCtx.lambda(true); + + iv($lambda); +} + function <> meta::external::query::sql::transformation::queryToPure::processArraySliceExpression(a:ArraySliceExpression[1], expContext:SqlTransformExpressionContext[1], context:SqlTransformContext[1]):ValueSpecification[1] { debug(|'processArraySliceExpression', $context.debug); diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/function_processors.pure b/legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/function_processors.pure index 2ed30dcddff..c4cd82dd325 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/function_processors.pure +++ b/legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/function_processors.pure @@ -535,8 +535,20 @@ function meta::external::query::sql::transformation::queryToPure::functionProces { let coreProcessor = $processors->filter(p | $p.name == $name->toLower()); - let processor = if ($coreProcessor->isEmpty(), + let udfProcessor = if ($coreProcessor->isEmpty(), | meta::external::query::sql::getSQLExtensions().udf($name), + | []); + + let udtfProcessor = if ($coreProcessor->isEmpty(), + | meta::external::query::sql::getSQLExtensions().udtf($name), + | []); + + let processor = if ($coreProcessor->isEmpty(), + | if ($udfProcessor->isEmpty(), + | if ($udtfProcessor->isEmpty(), + | [], + | $udtfProcessor), + | $udfProcessor), | $coreProcessor); assertEquals(1, $processor->size(), | 'No function matches the given name "' + $name + '"');