Add UDTF support for core_dataprofile

legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/pom.xml

@@ -160,6 +160,11 @@
                         <artifactId>legend-engine-xt-sql-pure</artifactId>
                         <version>${project.version}</version>
                     </dependency>
+                    <dependency>
+                        <groupId>org.finos.legend.engine</groupId>
+                        <artifactId>legend-engine-xt-relationalStore-postgresSqlModel-pure</artifactId>
+                        <version>${project.version}</version>
+                    </dependency>
                 </dependencies>
             </plugin>
         </plugins>
@@ -174,6 +179,10 @@
             <groupId>org.finos.legend.pure</groupId>
             <artifactId>legend-pure-m3-core</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-xt-relationalStore-postgresSqlModel-pure</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.finos.legend.pure</groupId>
             <artifactId>legend-pure-m2-dsl-graph-pure</artifactId>

legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality.definition.json

@@ -10,6 +10,7 @@
     "core_functions_standard",
     "core_functions_standard",
     "core_external_query_sql",
+    "core_external_store_relational_postgres_sql_model",
     "platform",
     "platform_dsl_store",
     "platform_dsl_graph",

legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/generation/dataqualitysqlextension.pure

@@ -0,0 +1,48 @@
+
+// Copyright 2026 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import meta::external::query::sql::*;
+import meta::external::query::sql::transformation::queryToPure::*;
+import meta::external::query::sql::metamodel::*;
+import meta::external::query::sql::transformation::utils::*;
+import meta::external::query::sql::transformation::compile::utils::*;
+import meta::external::dataquality::dataprofile::*;
+import meta::pure::metamodel::relation::*;
+import meta::external::query::sql::transformation::queryToPure::*;
+
+function <<sql.Extension>> meta::external::dataquality::dataProfileCoreExtension():SQLExtension[1]
+{
+  ^SQLExtension(
+    name = 'dq',
+    udtfs = ^UserDefinedTableFunctions(
+      prefix = 'core',
+      processors = [
+        misc('data_profile', 
+          meta::pure::metamodel::relation::Relation, 
+          {args, fc, expCtx, ctx |
+
+          assert($args->size() == 1, | 'expects single relational table');
+
+          let relationLambda = $args->at(0)->cast(@InstanceValue).values->cast(@LambdaFunction<Any>)->toOne();
+
+          let profilingLambda = iv(getProfilingLambda($relationLambda, [], [], false));
+
+          sfe(eval_Function_1__V_m_, $profilingLambda);
+          }
+        )
+      ]
+    )
+  )
+}

legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/test/testTranspile.pure

@@ -0,0 +1,143 @@
+// Copyright 2026 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+###Pure
+import meta::pure::functions::variant::navigation::*;
+import meta::pure::functions::variant::convert::*;
+import meta::pure::functions::hash::*;
+import meta::relational::extension::*;
+import meta::external::query::sql::transformation::queryToPure::*;
+import meta::external::query::sql::metamodel::*;
+import meta::external::query::sql::transformation::queryToPure::tests::*;
+import meta::external::query::sql::*;
+import meta::external::query::sql::schema::metamodel::*;
+import meta::legend::service::metamodel::*;
+import meta::pure::functions::meta::*;
+import meta::pure::mapping::*;
+import meta::pure::metamodel::serialization::grammar::*;
+import meta::external::query::sql::transformation::compile::utils::*;
+
+//SELECT
+function <<test.Test>>meta::external::dataquality::tests::testDataProfile():Boolean[1]
+{
+  test(
+    'SELECT * FROM core_data_profile((SELECT * FROM service."/service/service1"))',
+
+    [
+      {| eval(|let cte = meta::external::query::sql::transformation::queryToPure::tests::FlatInput.all()->project(
+            ~[ Boolean: x | $x.booleanIn, Integer: x | $x.integerIn, Float: x | $x.floatIn, Decimal: x | $x.decimalIn, StrictDate: x | $x.strictDateIn, DateTime: x | $x.dateTimeIn, String: x | $x.stringIn]
+          );
+          $cte->aggregate(~[
+            column_name: x | 'Boolean': y | 'Boolean',
+            count: x | $x.Boolean : y | $y->count(),
+            count_distinct: x | $x.Boolean : y | $y->distinct()->count(),
+            count_null: x | if($x.Boolean->isEmpty(), |1, |[]) : y | $y->count(),
+
+            'number/max': x | []->cast(@Number) : y | $y->max(),
+            'number/min': x | []->cast(@Number) : y | $y->min(),
+            'date/max': x | []->cast(@Date) : y | $y->max(),
+            'date/min': x | []->cast(@Date) : y | $y->min(),
+            'string/max_length': x | []->cast(@Integer) : y | $y->max(),
+            'string/min_length': x | []->cast(@Integer) : y | $y->min()
+            ])
+          ->concatenate(
+            $cte->aggregate(~[
+              column_name: x | 'Integer': y | 'Integer',
+              count: x | $x.Integer : y | $y->count(),
+              count_distinct: x | $x.Integer : y | $y->distinct()->count(),
+              count_null: x | if($x.Integer->isEmpty(), |1, |[]) : y | $y->count(),
+
+              'number/max': x | $x.Integer->cast(@Number) : y | $y->max(),
+              'number/min': x | $x.Integer->cast(@Number) : y | $y->min(),
+              'date/max': x | []->cast(@Date) : y | $y->max(),
+              'date/min': x | []->cast(@Date) : y | $y->min(),
+              'string/max_length': x | []->cast(@Integer) : y | $y->max(),
+              'string/min_length': x | []->cast(@Integer) : y | $y->min()
+              ])
+          )->concatenate(
+            $cte->aggregate(~[
+              column_name: x | 'Float': y | 'Float',
+              count: x | $x.Float : y | $y->count(),
+              count_distinct: x | $x.Float : y | $y->distinct()->count(),
+              count_null: x | if($x.Float->isEmpty(), |1, |[]) : y | $y->count(),
+
+              'number/max': x | $x.Float->cast(@Number) : y | $y->max(),
+              'number/min': x | $x.Float->cast(@Number) : y | $y->min(),
+              'date/max': x | []->cast(@Date) : y | $y->max(),
+              'date/min': x | []->cast(@Date) : y | $y->min(),
+              'string/max_length': x | []->cast(@Integer) : y | $y->max(),
+              'string/min_length': x | []->cast(@Integer) : y | $y->min()
+              ])
+         )->concatenate(
+            $cte->aggregate(~[
+              column_name: x | 'Decimal': y | 'Decimal',
+              count: x | $x.Decimal : y | $y->count(),
+              count_distinct: x | $x.Decimal : y | $y->distinct()->count(),
+              count_null: x | if($x.Decimal->isEmpty(), |1, |[]) : y | $y->count(),
+
+              'number/max': x | $x.Decimal->cast(@Number) : y | $y->max(),
+              'number/min': x | $x.Decimal->cast(@Number) : y | $y->min(),
+              'date/max': x | []->cast(@Date) : y | $y->max(),
+              'date/min': x | []->cast(@Date) : y | $y->min(),
+              'string/max_length': x | []->cast(@Integer) : y | $y->max(),
+              'string/min_length': x | []->cast(@Integer) : y | $y->min()
+              ])
+          )->concatenate(
+            $cte->aggregate(~[
+              column_name: x | 'StrictDate': y | 'StrictDate',
+              count: x | $x.StrictDate : y | $y->count(),
+              count_distinct: x | $x.StrictDate : y | $y->distinct()->count(),
+              count_null: x | if($x.StrictDate->isEmpty(), |1, |[]) : y | $y->count(),
+
+              'number/max': x | []->cast(@Number) : y | $y->max(),
+              'number/min': x | []->cast(@Number) : y | $y->min(),
+              'date/max': x | $x.StrictDate->cast(@Date) : y | $y->max(),
+              'date/min': x | $x.StrictDate->cast(@Date) : y | $y->min(),
+              'string/max_length': x | []->cast(@Integer) : y | $y->max(),
+              'string/min_length': x | []->cast(@Integer) : y | $y->min()
+              ])
+          )->concatenate(
+            $cte->aggregate(~[
+              column_name: x | 'DateTime': y | 'DateTime',
+              count: x | $x.DateTime : y | $y->count(),
+              count_distinct: x | $x.DateTime : y | $y->distinct()->count(),
+              count_null: x | if($x.DateTime->isEmpty(), |1, |[]) : y | $y->count(),
+
+              'number/max': x | []->cast(@Number) : y | $y->max(),
+              'number/min': x | []->cast(@Number) : y | $y->min(),
+              'date/max': x | $x.DateTime->cast(@Date) : y | $y->max(),
+              'date/min': x | $x.DateTime->cast(@Date) : y | $y->min(),
+              'string/max_length': x | []->cast(@Integer) : y | $y->max(),
+              'string/min_length': x | []->cast(@Integer) : y | $y->min()
+              ])
+          )->concatenate(
+            $cte->aggregate(~[
+              column_name: x | 'String': y | 'String',
+              count: x | $x.String : y | $y->count(),
+              count_distinct: x | $x.String : y | $y->distinct()->count(),
+              count_null: x | if($x.String->isEmpty(), |1, |[]) : y | $y->count(),
+
+              'number/max': x | []->cast(@Number) : y | $y->max(),
+              'number/min': x | []->cast(@Number) : y | $y->min(),
+              'date/max': x | []->cast(@Date) : y | $y->max(),
+              'date/min': x | []->cast(@Date) : y | $y->min(),
+              'string/max_length': x | $x.String->length() : y | $y->max(),
+              'string/min_length': x | $x.String->length() : y | $y->min()
+              ])
+          );
+        )
+      }
+    ]
+  )
+}

legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-postgresSql/legend-engine-xt-relationalStore-postgresSql-grammar/src/main/java/org/finos/legend/engine/language/sql/grammar/from/SqlVisitor.java

@@ -1522,8 +1522,12 @@ public Node visitNestedExpression(SqlBaseParser.NestedExpressionContext context)
     @Override
     public Node visitSubqueryExpression(SqlBaseParser.SubqueryExpressionContext context)
     {
-//        return new SubqueryExpression((Query) visit(context.query()));
-        return unsupported("Subquery Expression");
+        Query query = (Query) visit(context.queryStatement());
+
+        SubqueryExpression expr = new SubqueryExpression();
+        expr.query = query;
+
+        return expr;
     }
 
     @Override

legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-postgresSql/legend-engine-xt-relationalStore-postgresSql-grammar/src/test/java/org/finos/legend/engine/language/sql/grammar/test/roundtrip/TestSQLRoundTrip.java

@@ -369,6 +369,9 @@ public void testNested()
         check("SELECT * from (SELECT col from myTable)");
     }
 
+    @Test
+    public void testTableSubQuery() {check("SELECT * from ((SELECT * from myTable))");}
+
     @Test
     public void testCommonTableExpressionSingle()
     {

legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/extension.pure

@@ -18,10 +18,15 @@ Class meta::external::query::sql::SQLExtension
 {
    name:String[1];
    udfs: meta::external::query::sql::UserDefinedFunctions[0..1];
+   udtfs: meta::external::query::sql::UserDefinedTableFunctions[0..1];
 
    udf(name:String[1]){
     $this.udfs.processors->filter(p | toLower($this.udfs->toOne().prefix + '_' + $p.name) == $name->toLower())
    }:meta::external::query::sql::transformation::queryToPure::FunctionProcessor[*];
+
+   udtf(name:String[1]){
+    $this.udtfs.processors->filter(p | toLower($this.udtfs->toOne().prefix + '_' + $p.name) == $name->toLower())
+   }:meta::external::query::sql::transformation::queryToPure::FunctionProcessor[*];
 }
 
 Class meta::external::query::sql::UserDefinedFunctions[
@@ -37,6 +42,19 @@ Class meta::external::query::sql::UserDefinedFunctions[
   }:String[*];
 }
 
+Class meta::external::query::sql::UserDefinedTableFunctions[
+  prefixNotBlank: $this.prefix->length() > 0,
+  prefixOnlyChars: $this.prefix->matches('[a-zA-Z]*')
+]
+{
+  prefix: String[1];
+  processors: meta::external::query::sql::transformation::queryToPure::FunctionProcessor[*];
+
+  names(){
+    $this.processors->map(p | $this.prefix + '_' + $p.name)
+  }:String[*];
+}
+
 Profile meta::external::query::sql::sql
 {
    stereotypes: [Extension];

legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/fromPure.pure

@@ -32,6 +32,7 @@ import meta::pure::executionPlan::*;
 import meta::external::query::sql::transformation::utils::*;
 import meta::external::query::sql::transformation::compile::utils::*;
 import meta::pure::functions::relation::*;
+import meta::pure::functions::meta::*;
 
 Class meta::external::query::sql::transformation::queryToPure::SQLSourceArgument
 {
@@ -2098,6 +2099,7 @@ function <<access.private>> meta::external::query::sql::transformation::queryToP
     s:SimpleCaseExpression[1] | processSimpleCaseExpression($s, $expContext, $context),
     s:SearchedCaseExpression[1] | processSearchedCaseExpression($s, $expContext, $context),
     s:SubscriptExpression[1] | processSubscriptExpression($s, $expContext, $context),
+    s:SubqueryExpression[1] | processSubqueryExpression($s.query, $context),
     t:Trim[1] | processTrim($t, $expContext, $context),
     e:meta::external::query::sql::metamodel::Expression[*] | fail('Expression type not yet supported'); iv(1);
   ])->evaluateAndDeactivate();
@@ -2556,6 +2558,15 @@ function <<access.private>> meta::external::query::sql::transformation::queryToP
   sfe(at_T_MANY__Integer_1__T_1_, [$value, $index]);
 }
 
+function <<access.private>> meta::external::query::sql::transformation::queryToPure::processSubqueryExpression(query: Query[1], context:SqlTransformContext[1]):ValueSpecification[1]
+{
+  let subQueryCtx = processQuery($query, $context);
+
+  let lambda = $subQueryCtx.lambda(true);
+
+  iv($lambda);
+}
+
 function <<access.private>> meta::external::query::sql::transformation::queryToPure::processArraySliceExpression(a:ArraySliceExpression[1], expContext:SqlTransformExpressionContext[1], context:SqlTransformContext[1]):ValueSpecification[1]
 {
   debug(|'processArraySliceExpression', $context.debug);

legend-engine-xts-sql/legend-engine-xt-sql-transformation/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/function_processors.pure

@@ -535,8 +535,20 @@ function meta::external::query::sql::transformation::queryToPure::functionProces
 {
   let coreProcessor = $processors->filter(p | $p.name == $name->toLower());
 
-  let processor = if ($coreProcessor->isEmpty(),
+  let udfProcessor = if ($coreProcessor->isEmpty(),
   | meta::external::query::sql::getSQLExtensions().udf($name),
+  | []);
+
+  let udtfProcessor = if ($coreProcessor->isEmpty(),
+  | meta::external::query::sql::getSQLExtensions().udtf($name),
+  | []);
+
+    let processor = if ($coreProcessor->isEmpty(),
+  | if ($udfProcessor->isEmpty(),
+    | if ($udtfProcessor->isEmpty(),
+    | [],
+    | $udtfProcessor),
+  | $udfProcessor),
   | $coreProcessor);
 
   assertEquals(1, $processor->size(), | 'No function matches the given name "' + $name + '"');