New IR -- WIP #24466

kasiafi · 2024-12-13T08:10:11Z

See core/trino-main/src/main/java/io/trino/sql/newir/README.md for details.

kasiafi · 2024-12-13T17:47:01Z

example assembly printout

IR version = 1
%0 = query() : () -> "boolean" ({
    ^query
        %1 = table_scan() : () -> "multiset(row(""f_1"" varchar(25),""f_2"" bigint))" ()
            {table_handle = "{""catalogHandle"":""tpch:normal:default"",""connectorHandle"":{""@type"":""../../plugin/trino-tpch/pom.xml:io.trino.plugin.tpch.TpchTableHandle"",""schemaName"":""tiny"",""tableName"":""nation"",""scaleFactor"":0.01,""constraint"":{""columnDomains"":[]}},""transaction"":[""../../plugin/trino-tpch/pom.xml:io.trino.plugin.tpch.TpchTransactionHandle"",""INSTANCE""]}", column_handles = "[{""@type"":""../../plugin/trino-tpch/pom.xml:io.trino.plugin.tpch.TpchColumnHandle"",""columnName"":""name"",""type"":""varchar(25)""},{""@type"":""../../plugin/trino-tpch/pom.xml:io.trino.plugin.tpch.TpchColumnHandle"",""columnName"":""regionkey"",""type"":""bigint""}]", constraint = "{""columnDomains"":[]}", update_target = "false", use_connector_node_partitioning = "false"}
        %2 = filter(%1) : ("multiset(row(""f_1"" varchar(25),""f_2"" bigint))") -> "multiset(row(""f_1"" varchar(25),""f_2"" bigint))" ({
            ^predicate (%3 : "row(""f_1"" varchar(25),""f_2"" bigint)")
                %4 = field_selection(%3) : ("row(""f_1"" varchar(25),""f_2"" bigint)") -> "bigint" ()
                    {field_name = "f_2"}
                %5 = constant() : () -> "bigint" ()
                    {constant_result = "{""type"":""bigint"",""value"":2}"}
                %6 = comparison(%4, %5) : ("bigint", "bigint") -> "boolean" ()
                    {comparison_operator = "GREATER_THAN"}
                %7 = return(%6) : ("boolean") -> "boolean" ()
                    {ir.terminal = "true"}
            })
        %8 = project(%2) : ("multiset(row(""f_1"" varchar(25),""f_2"" bigint))") -> "multiset(row(""f_1"" varchar(25)))" ({
            ^assignments (%9 : "row(""f_1"" varchar(25),""f_2"" bigint)")
                %10 = field_selection(%9) : ("row(""f_1"" varchar(25),""f_2"" bigint)") -> "varchar(25)" ()
                    {field_name = "f_1"}
                %11 = call(%10) : ("varchar(25)") -> "varchar(25)" ()
                    {resolved_function = "{""signature"":{""name"":{""catalogName"":""system"",""schemaName"":""builtin"",""functionName"":""lower""},""returnType"":""varchar(25)"",""argumentTypes"":[""varchar(25)""]},""catalogHandle"":""system:normal:system"",""functionId"":""lower(varchar(x)):varchar(x)"",""functionKind"":""SCALAR"",""deterministic"":true,""functionNullability"":{""returnNullable"":false,""argumentNullable"":[false]},""typeDependencies"":{},""functionDependencies"":[]}"}
                %12 = row(%11) : ("varchar(25)") -> "row(varchar(25))" ()
                %13 = return(%12) : ("row(varchar(25))") -> "row(varchar(25))" ()
                    {ir.terminal = "true"}
            })
        %14 = output(%8) : ("multiset(row(""f_1"" varchar(25)))") -> "boolean" ({
            ^outputFieldSelector (%15 : "row(""f_1"" varchar(25))")
                %16 = field_selection(%15) : ("row(""f_1"" varchar(25))") -> "varchar(25)" ()
                    {field_name = "f_1"}
                %17 = row(%16) : ("varchar(25)") -> "row(varchar(25))" ()
                %18 = return(%17) : ("row(varchar(25))") -> "row(varchar(25))" ()
                    {ir.terminal = "true"}
            })
            {output_names = "[""_col0""]", ir.terminal = "true"}
    })
    {ir.terminal = "true"}

wendigo · 2025-02-13T11:58:42Z

core/trino-parser/src/main/java/io/trino/newir/tree/OperationNode.java

+{
+    public OperationNode(Optional<String> dialect, String name, String resultName, TypeNode resultType, List<String> argumentNames, List<TypeNode> argumentTypes, List<RegionNode> regions, List<AttributeNode> attributes)
+    {
+        requireNonNull(dialect, "dialect is null");
+        requireNonNull(name, "name is null");
+        requireNonNull(resultName, "resultName is null");
+        requireNonNull(resultType, "resultType is null");
+        requireNonNull(argumentNames, "argumentNames is null");
+        requireNonNull(argumentTypes, "argumentTypes is null");
+        requireNonNull(regions, "regions is null");
+        requireNonNull(attributes, "attributes is null");
+
+        checkArgument(argumentNames.size() == argumentTypes.size(), "argument names and argument types lists do not match in size");
+
+        this.dialect = dialect;
+        this.name = name;
+        this.resultName = resultName;
+        this.resultType = resultType;
+        this.argumentNames = ImmutableList.copyOf(argumentNames);
+        this.argumentTypes = ImmutableList.copyOf(argumentTypes);
+        this.regions = ImmutableList.copyOf(regions);
+        this.attributes = ImmutableList.copyOf(attributes);
+    }


Suggested change

{

public OperationNode(Optional<String> dialect, String name, String resultName, TypeNode resultType, List<String> argumentNames, List<TypeNode> argumentTypes, List<RegionNode> regions, List<AttributeNode> attributes)

{

requireNonNull(dialect, "dialect is null");

requireNonNull(name, "name is null");

requireNonNull(resultName, "resultName is null");

requireNonNull(resultType, "resultType is null");

requireNonNull(argumentNames, "argumentNames is null");

requireNonNull(argumentTypes, "argumentTypes is null");

requireNonNull(regions, "regions is null");

requireNonNull(attributes, "attributes is null");

checkArgument(argumentNames.size() == argumentTypes.size(), "argument names and argument types lists do not match in size");

this.dialect = dialect;

this.name = name;

this.resultName = resultName;

this.resultType = resultType;

this.argumentNames = ImmutableList.copyOf(argumentNames);

this.argumentTypes = ImmutableList.copyOf(argumentTypes);

this.regions = ImmutableList.copyOf(regions);

this.attributes = ImmutableList.copyOf(attributes);

}

{

public OperationNode

{

requireNonNull(dialect, "dialect is null");

requireNonNull(name, "name is null");

requireNonNull(resultName, "resultName is null");

requireNonNull(resultType, "resultType is null");

requireNonNull(argumentNames, "argumentNames is null");

requireNonNull(argumentTypes, "argumentTypes is null");

requireNonNull(regions, "regions is null");

requireNonNull(attributes, "attributes is null");

checkArgument(argumentNames.size() == argumentTypes.size(), "argument names and argument types lists do not match in size");

argumentNames = ImmutableList.copyOf(argumentNames);

argumentTypes = ImmutableList.copyOf(argumentTypes);

regions = ImmutableList.copyOf(regions);

attributes = ImmutableList.copyOf(attributes);

}

}

martint · 2025-02-13T20:01:43Z

core/trino-parser/src/main/java/io/trino/newir/tree/AttributeNode.java

+
+import static java.util.Objects.requireNonNull;
+
+public record AttributeNode(Optional<String> dialect, String name, String attribute)


String value

martint · 2025-02-13T20:03:53Z

core/trino-parser/src/main/java/io/trino/newir/tree/BlockNode.java

+import static com.google.common.base.Preconditions.checkArgument;
+import static java.util.Objects.requireNonNull;
+
+public record BlockNode(Optional<String> name, List<String> parameterNames, List<TypeNode> parameterTypes, List<OperationNode> operations)


Instead of two parallel lists (parameterNames and parameterTypes), use a single list of a composite object type (e.g., Parameter) that contains a name and a type

martint · 2025-02-13T20:15:43Z

core/trino-parser/src/main/java/io/trino/newir/tree/OperationNode.java

+import static com.google.common.base.Preconditions.checkArgument;
+import static java.util.Objects.requireNonNull;
+
+public record OperationNode(


Flattening everything makes it harder to understand the relationship between the elements. I would structure it such that the operation has:

a dialect

a name

a list of arguments

an operation type (which in turn, has a result type and argument types)

a list of regions

a list of attributes

Also, we may need to model assignments as an element enclosing an operation, since not all callsites may need to be assigned. I.e., something like:

assignment ::= name '=' operation

martint · 2025-02-13T20:37:21Z

core/trino-main/src/main/java/io/trino/sql/newir/Value.java

+
+    static void validateValueName(String name)
+    {
+        if (!name.startsWith("%") || !isValidPrefixedIdentifier(name.substring(1))) {


The '%' prefix should only exist at parsing and printing time, but should not be carried over in the name of the variable.

martint · 2025-02-13T20:46:58Z

core/trino-main/src/main/java/io/trino/sql/newir/FormatOptions.java

+public class FormatOptions
+{
+    public static final String INDENT = "    ";
+    private static final Pattern IDENTIFIER = Pattern.compile("([a-z]|[A-Z]|_)([a-z]|[A-Z]|[0-9]|_)*");


[a-z]|[A-Z]|[0-9]|_ can be simplified to [a-zA-Z0-9_]

martint · 2025-02-13T21:19:47Z

core/trino-main/src/main/java/io/trino/sql/dialect/trino/operation/In.java

+    private final List<Value> inputList;
+    private final Map<AttributeKey, Object> attributes;
+
+    public In(String resultName, Value input, List<Value> inputList, List<Map<AttributeKey, Object>> sourceAttributes)


The input list should modeled as a single value of type "array"

martint · 2025-02-21T01:27:28Z

core/trino-main/src/main/java/io/trino/sql/dialect/trino/Context.java

+    public Context(Block.Builder block, Map<Symbol, RowField> symbolMapping)
+    {
+        this.block = requireNonNull(block, "block is null");
+        this.symbolMapping = ImmutableMap.copyOf(requireNonNull(symbolMapping, "symbolMapping is null"));


requireNonNull not needed

martint · 2025-02-21T01:53:22Z

core/trino-main/src/main/java/io/trino/sql/newir/Program.java

+    public Operation getOperation(Result value)
+    {
+        SourceNode source = valueMap.get(value);
+        if (source == null) {
+            throw new TrinoException(IR_ERROR, format("value %s is not defined", value.name()));
+        }
+
+        if (source instanceof Operation operation) {
+            if (!value.type().equals(operation.result().type())) {
+                throw new TrinoException(IR_ERROR, format("value %s type mismatch. expected: %s, actual: %s", value.name(), value.type(), operation.result().type()));
+            }
+            return operation;
+        }
+
+        throw new TrinoException(IR_ERROR, format("value %s is not an operation result", value.name()));
+    }
+
+    public Block getBlock(Parameter value)
+    {
+        SourceNode source = valueMap.get(value);
+        if (source == null) {
+            throw new TrinoException(IR_ERROR, format("value %s is not defined", value.name()));
+        }
+
+        if (source instanceof Block block) {
+            Type parameterType = block.parameters().get(block.getIndex(value)).type();
+            if (!value.type().equals(parameterType)) {
+                throw new TrinoException(IR_ERROR, format("value %s type mismatch. expected: %s, actual: %s", value.name(), value.type(), parameterType));
+            }
+            return block;
+        }
+
+        throw new TrinoException(IR_ERROR, format("value %s is not a block parameter", value.name()));
+    }


These methods aren't used anywhere. What's the use case for them?

github-actions · 2025-04-16T17:03:49Z

This pull request has gone a while without any activity. Ask for help on #core-dev on Trino slack.

cla-bot bot added the cla-signed label Dec 13, 2024

kasiafi force-pushed the 526Abstractions branch 2 times, most recently from ed4c70f to 6ea030e Compare December 13, 2024 17:44

kasiafi force-pushed the 526Abstractions branch 14 times, most recently from 30ed4b3 to 852f33b Compare December 20, 2024 15:17

kasiafi force-pushed the 526Abstractions branch 12 times, most recently from d71a5b9 to 8f19d4a Compare December 29, 2024 10:53

kasiafi force-pushed the 526Abstractions branch from 27649af to 40aeed5 Compare February 7, 2025 19:41

wendigo reviewed Feb 13, 2025

View reviewed changes

martint reviewed Feb 21, 2025

View reviewed changes

github-actions bot added the iceberg Iceberg connector label Feb 25, 2025

kasiafi force-pushed the 526Abstractions branch from 6a1228b to 3fb3ccd Compare February 25, 2025 19:40

kasiafi force-pushed the 526Abstractions branch 7 times, most recently from d53a39e to 6c59149 Compare March 20, 2025 15:22

kasiafi marked this pull request as draft March 20, 2025 16:50

kasiafi force-pushed the 526Abstractions branch 5 times, most recently from 227b659 to 909c600 Compare March 25, 2025 13:32

kasiafi added 2 commits March 25, 2025 16:18

New IR -- abstractions and dialects

3444d4a

Use anonymous row as intermediate relation row type

7a81127

kasiafi force-pushed the 526Abstractions branch 2 times, most recently from b8e5df6 to eeb711c Compare March 26, 2025 11:15

kasiafi added 4 commits March 26, 2025 16:16

Fix Output operation: consider the spooling metadata field

a98e7d3

Add parser for new IR

d03c40e

Test IR print with QueryRunner

aa3e5ae

CTE reuse WIP

c0ab3dc

kasiafi force-pushed the 526Abstractions branch from eeb711c to c0ab3dc Compare March 26, 2025 15:16

github-actions bot added the stale label Apr 16, 2025

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

New IR -- WIP #24466

New IR -- WIP #24466

kasiafi commented Dec 13, 2024 •

edited

Loading

kasiafi commented Dec 13, 2024 •

edited

Loading

wendigo Feb 13, 2025

martint Feb 13, 2025

martint Feb 13, 2025

martint Feb 13, 2025

martint Feb 13, 2025

martint Feb 13, 2025

martint Feb 13, 2025

martint Feb 21, 2025

martint Feb 21, 2025

github-actions bot commented Apr 16, 2025


		import static java.util.Objects.requireNonNull;

		public record AttributeNode(Optional<String> dialect, String name, String attribute)

New IR -- WIP #24466

Are you sure you want to change the base?

New IR -- WIP #24466

Conversation

kasiafi commented Dec 13, 2024 • edited Loading

kasiafi commented Dec 13, 2024 • edited Loading

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

github-actions bot commented Apr 16, 2025

kasiafi commented Dec 13, 2024 •

edited

Loading

kasiafi commented Dec 13, 2024 •

edited

Loading