New IR -- WIP

core/trino-grammar/src/main/antlr4/io/trino/grammar/newir/NewIr.g4

@@ -0,0 +1,120 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+grammar NewIr;
+
+tokens {
+    DELIMITER
+}
+
+program
+    : IR VERSION EQ version=INTEGER_VALUE
+        operation EOF
+    ;
+
+operation
+    : resultName=VALUE_NAME EQ operationName
+        '(' (argumentNames+=VALUE_NAME (',' argumentNames+=VALUE_NAME)*)? ')'
+        ':' '(' (argumentTypes+=type (',' argumentTypes+=type)*)? ')'
+        '->' resultType=type
+        '(' (region (',' region)*)? ')'
+        ('{' (attribute (',' attribute)*)? '}')? // does not roundtrip: we don't print empty attributes list // TODO test
+    ;
+
+region
+    : '{' block+ '}'
+    ;
+
+block
+    : BLOCK_NAME?
+        ('(' blockParameter (',' blockParameter)* ')')?
+        operation+
+    ;
+
+blockParameter
+    : VALUE_NAME ':' type
+    ;
+
+attribute
+    : attributeName EQ STRING
+    ;
+
+identifier
+    : IDENTIFIER
+    | nonReserved
+    ;
+
+dialectName
+    : identifier
+    ;
+
+operationName
+    : (dialectName '.')? identifier
+    ;
+
+attributeName
+    : (dialectName '.')? identifier
+    ;
+
+type
+    : (dialectName '.')? STRING
+    ;
+
+nonReserved
+    : IR | VERSION
+    ;
+
+IR: 'IR';
+VERSION: 'version';
+
+EQ: '=';
+
+STRING
+    : '"' ( ~'"' | '""' )* '"'
+    ;
+
+VALUE_NAME
+    : '%' PREFIXED_IDENTIFIER
+    ;
+
+BLOCK_NAME
+    : '^' PREFIXED_IDENTIFIER
+    ;
+
+INTEGER_VALUE
+    : DIGIT+
+    ;
+
+IDENTIFIER
+    : (LETTER | '_') (LETTER | DIGIT | '_')*
+    ;
+
+PREFIXED_IDENTIFIER
+    : (LETTER | DIGIT | '_')+
+    ;
+
+fragment DIGIT
+    : [0-9]
+    ;
+
+fragment LETTER
+    : [a-z] | [A-Z]
+    ;
+
+WS
+    : [ \r\n\t]+ -> channel(HIDDEN)
+    ;
+
+// Catch-all for anything we can't recognize.
+UNRECOGNIZED: .;

core/trino-main/src/main/java/io/trino/execution/SqlQueryExecution.java

@@ -61,6 +61,8 @@
 import io.trino.sql.analyzer.Analysis;
 import io.trino.sql.analyzer.Analyzer;
 import io.trino.sql.analyzer.AnalyzerFactory;
+import io.trino.sql.newir.FormatOptions;
+import io.trino.sql.newir.Program;
 import io.trino.sql.planner.AdaptivePlanner;
 import io.trino.sql.planner.InputExtractor;
 import io.trino.sql.planner.LogicalPlanner;
@@ -74,6 +76,7 @@
 import io.trino.sql.planner.SubPlan;
 import io.trino.sql.planner.optimizations.AdaptivePlanOptimizer;
 import io.trino.sql.planner.optimizations.PlanOptimizer;
+import io.trino.sql.planner.optimizations.ctereuse.CteReuse;
 import io.trino.sql.planner.plan.OutputNode;
 import io.trino.sql.tree.ExplainAnalyze;
 import io.trino.sql.tree.Query;
@@ -148,6 +151,7 @@ public class SqlQueryExecution
     private final EventDrivenTaskSourceFactory eventDrivenTaskSourceFactory;
     private final TaskDescriptorStorage taskDescriptorStorage;
     private final PlanOptimizersStatsCollector planOptimizersStatsCollector;
+    private final FormatOptions formatOptions;
 
     private SqlQueryExecution(
             PreparedQuery preparedQuery,
@@ -185,7 +189,8 @@ private SqlQueryExecution(
             SqlTaskManager coordinatorTaskManager,
             ExchangeManagerRegistry exchangeManagerRegistry,
             EventDrivenTaskSourceFactory eventDrivenTaskSourceFactory,
-            TaskDescriptorStorage taskDescriptorStorage)
+            TaskDescriptorStorage taskDescriptorStorage,
+            FormatOptions formatOptions)
     {
         try (SetThreadName _ = new SetThreadName("Query-" + stateMachine.getQueryId())) {
             this.slug = requireNonNull(slug, "slug is null");
@@ -240,6 +245,7 @@ private SqlQueryExecution(
             this.eventDrivenTaskSourceFactory = requireNonNull(eventDrivenTaskSourceFactory, "taskSourceFactory is null");
             this.taskDescriptorStorage = requireNonNull(taskDescriptorStorage, "taskDescriptorStorage is null");
             this.planOptimizersStatsCollector = requireNonNull(planOptimizersStatsCollector, "planOptimizersStatsCollector is null");
+            this.formatOptions = requireNonNull(formatOptions, "formatOptions is null");
         }
     }
 
@@ -503,6 +509,9 @@ private PlanRoot doPlanQuery(CachingTableStatsProvider tableStatsProvider)
         Plan plan = logicalPlanner.plan(analysis);
         queryPlan.set(plan);
 
+        Optional<Program> optimizedProgram = CteReuse.reuseCommonSubqueries(plan, plannerContext, getSession(), formatOptions);
+        checkState(optimizedProgram.isEmpty());
+
         // fragment the plan
         SubPlan fragmentedPlan;
         try (var _ = scopedSpan(tracer, "fragment-plan")) {
@@ -809,6 +818,7 @@ public static class SqlQueryExecutionFactory
         private final ExchangeManagerRegistry exchangeManagerRegistry;
         private final EventDrivenTaskSourceFactory eventDrivenTaskSourceFactory;
         private final TaskDescriptorStorage taskDescriptorStorage;
+        private final FormatOptions formatOptions;
 
         @Inject
         SqlQueryExecutionFactory(
@@ -841,7 +851,8 @@ public static class SqlQueryExecutionFactory
                 SqlTaskManager coordinatorTaskManager,
                 ExchangeManagerRegistry exchangeManagerRegistry,
                 EventDrivenTaskSourceFactory eventDrivenTaskSourceFactory,
-                TaskDescriptorStorage taskDescriptorStorage)
+                TaskDescriptorStorage taskDescriptorStorage,
+                FormatOptions formatOptions)
         {
             this.tracer = requireNonNull(tracer, "tracer is null");
             this.schedulerStats = requireNonNull(schedulerStats, "schedulerStats is null");
@@ -875,6 +886,7 @@ public static class SqlQueryExecutionFactory
             this.exchangeManagerRegistry = requireNonNull(exchangeManagerRegistry, "exchangeManagerRegistry is null");
             this.eventDrivenTaskSourceFactory = requireNonNull(eventDrivenTaskSourceFactory, "eventDrivenTaskSourceFactory is null");
             this.taskDescriptorStorage = requireNonNull(taskDescriptorStorage, "taskDescriptorStorage is null");
+            this.formatOptions = requireNonNull(formatOptions, "formatOptions is null");
         }
 
         @Override
@@ -925,7 +937,8 @@ public QueryExecution createQueryExecution(
                     coordinatorTaskManager,
                     exchangeManagerRegistry,
                     eventDrivenTaskSourceFactory,
-                    taskDescriptorStorage);
+                    taskDescriptorStorage,
+                    formatOptions);
         }
     }
 }

core/trino-main/src/main/java/io/trino/metadata/Metadata.java

@@ -54,6 +54,7 @@
 import io.trino.spi.connector.TableFunctionApplicationResult;
 import io.trino.spi.connector.TableScanRedirectApplicationResult;
 import io.trino.spi.connector.TopNApplicationResult;
+import io.trino.spi.connector.UnificationResult;
 import io.trino.spi.connector.WriterScalingOptions;
 import io.trino.spi.expression.ConnectorExpression;
 import io.trino.spi.expression.Constant;
@@ -569,6 +570,8 @@ Optional<TopNApplicationResult<TableHandle>> applyTopN(
 
     Optional<TableFunctionApplicationResult<TableHandle>> applyTableFunction(Session session, TableFunctionHandle handle);
 
+    Optional<UnificationResult<TableHandle>> unifyTables(Session session, TableHandle first, TableHandle second);
+
     default void validateScan(Session session, TableHandle table) {}
 
     //

core/trino-main/src/main/java/io/trino/metadata/MetadataManager.java

@@ -88,6 +88,7 @@
 import io.trino.spi.connector.TableFunctionApplicationResult;
 import io.trino.spi.connector.TableScanRedirectApplicationResult;
 import io.trino.spi.connector.TopNApplicationResult;
+import io.trino.spi.connector.UnificationResult;
 import io.trino.spi.connector.WriterScalingOptions;
 import io.trino.spi.expression.ConnectorExpression;
 import io.trino.spi.expression.Constant;
@@ -2157,6 +2158,24 @@ public Optional<TableFunctionApplicationResult<TableHandle>> applyTableFunction(
                         result.getColumnHandles()));
     }
 
+    @Override
+    public Optional<UnificationResult<TableHandle>> unifyTables(Session session, TableHandle first, TableHandle second)
+    {
+        CatalogHandle catalogHandle = first.catalogHandle();
+        ConnectorTransactionHandle transaction = first.transaction();
+        if (!catalogHandle.equals(second.catalogHandle()) || !transaction.equals(second.transaction())) {
+            return Optional.empty();
+        }
+        ConnectorMetadata metadata = getMetadata(session, catalogHandle);
+
+        return metadata.unifyTables(session.toConnectorSession(catalogHandle), first.connectorHandle(), second.connectorHandle())
+                .map(result -> new UnificationResult<>(
+                        new TableHandle(catalogHandle, result.unifiedHandle(), transaction),
+                        result.firstCompensationFilter(),
+                        result.secondCompensationFilter(),
+                        result.enforcedProperties()));
+    }
+
     private void verifyProjection(TableHandle table, List<ConnectorExpression> projections, List<Assignment> assignments, int expectedProjectionSize)
     {
         projections.forEach(projection -> requireNonNull(projection, "one of the projections is null"));

core/trino-main/src/main/java/io/trino/server/ServerMainModule.java

@@ -19,6 +19,7 @@
 import com.google.inject.Provides;
 import com.google.inject.Scopes;
 import com.google.inject.Singleton;
+import com.google.inject.TypeLiteral;
 import com.google.inject.multibindings.ProvidesIntoSet;
 import io.airlift.concurrent.BoundedExecutor;
 import io.airlift.configuration.AbstractConfigurationAwareModule;
@@ -82,6 +83,7 @@
 import io.trino.metadata.SystemFunctionBundle;
 import io.trino.metadata.SystemSecurityMetadata;
 import io.trino.metadata.TableFunctionRegistry;
+import io.trino.metadata.TableHandle;
 import io.trino.metadata.TableProceduresRegistry;
 import io.trino.metadata.TypeRegistry;
 import io.trino.operator.DirectExchangeClientConfig;
@@ -109,6 +111,9 @@
 import io.trino.spi.VersionEmbedder;
 import io.trino.spi.block.Block;
 import io.trino.spi.block.BlockEncodingSerde;
+import io.trino.spi.connector.ColumnHandle;
+import io.trino.spi.predicate.NullableValue;
+import io.trino.spi.predicate.TupleDomain;
 import io.trino.spi.type.Type;
 import io.trino.spi.type.TypeManager;
 import io.trino.spi.type.TypeOperators;
@@ -131,18 +136,23 @@
 import io.trino.sql.SqlEnvironmentConfig;
 import io.trino.sql.analyzer.SessionTimeProvider;
 import io.trino.sql.analyzer.StatementAnalyzerFactory;
+import io.trino.sql.dialect.trino.TrinoAttributeRegistry;
+import io.trino.sql.dialect.trino.TrinoDialect;
 import io.trino.sql.gen.CursorProcessorCompiler;
 import io.trino.sql.gen.ExpressionCompiler;
 import io.trino.sql.gen.JoinCompiler;
 import io.trino.sql.gen.JoinFilterFunctionCompiler;
 import io.trino.sql.gen.OrderingCompiler;
 import io.trino.sql.gen.PageFunctionCompiler;
 import io.trino.sql.gen.columnar.ColumnarFilterCompiler;
+import io.trino.sql.newir.DialectRegistry;
+import io.trino.sql.newir.FormatOptions;
 import io.trino.sql.parser.SqlParser;
 import io.trino.sql.planner.CompilerConfig;
 import io.trino.sql.planner.LocalExecutionPlanner;
 import io.trino.sql.planner.NodePartitioningManager;
 import io.trino.sql.planner.OptimizerConfig;
+import io.trino.sql.planner.PartitioningHandle;
 import io.trino.sql.planner.RuleStatsRecorder;
 import io.trino.sql.planner.Symbol;
 import io.trino.sql.planner.SymbolKeyDeserializer;
@@ -158,6 +168,7 @@
 import io.trino.util.EmbedVersion;
 import io.trino.util.FinalizerService;
 
+import java.util.List;
 import java.util.Set;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ExecutorService;
@@ -427,6 +438,18 @@ protected void setup(Binder binder)
         newSetBinder(binder, Type.class);
         binder.bind(RegisterJsonPath2016Type.class).asEagerSingleton();
 
+        // new IR
+        jsonCodecBinder(binder).bindJsonCodec(TableHandle.class);
+        jsonCodecBinder(binder).bindJsonCodec(new TypeLiteral<List<ColumnHandle>>() {});
+        jsonCodecBinder(binder).bindJsonCodec(new TypeLiteral<TupleDomain<ColumnHandle>>() {});
+        jsonCodecBinder(binder).bindJsonCodec(PartitioningHandle.class);
+        jsonCodecBinder(binder).bindJsonCodec(NullableValue.class);
+        jsonCodecBinder(binder).bindJsonCodec(new TypeLiteral<NullableValue[]>() {});
+        binder.bind(TrinoAttributeRegistry.class).in(Scopes.SINGLETON);
+        binder.bind(TrinoDialect.class).in(Scopes.SINGLETON);
+        binder.bind(DialectRegistry.class).in(Scopes.SINGLETON);
+        binder.bind(FormatOptions.class).in(Scopes.SINGLETON);
+
         // split manager
         binder.bind(SplitManager.class).in(Scopes.SINGLETON);