diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java index c1d9c11660993..56383aca916dc 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java @@ -166,6 +166,13 @@ protected QueryRunner createQueryRunner() Optional.empty()); } + @Override + protected QueryRunner createExpectedQueryRunner() + throws Exception + { + return getQueryRunner(); + } + @Test public void testMetadataQueryOptimizationWithLimit() { @@ -1366,6 +1373,18 @@ public void testPushdownSubfields() assertPushdownSubfields("SELECT x.a FROM test_pushdown_struct_subfields WHERE x.a > 10 AND x.b LIKE 'abc%'", "test_pushdown_struct_subfields", ImmutableMap.of("x", toSubfields("x.a", "x.b"))); + assertQuery("SELECT struct.b FROM (SELECT CUSTOM_STRUCT_WITH_PASSTHROUGH(x) AS struct FROM test_pushdown_struct_subfields)"); + assertQuery("SELECT struct.b FROM (SELECT CUSTOM_STRUCT_WITHOUT_PASSTHROUGH(x) AS struct FROM test_pushdown_struct_subfields)"); + + assertPushdownSubfields("SELECT struct.b FROM (SELECT CUSTOM_STRUCT_WITH_PASSTHROUGH(x) AS struct FROM test_pushdown_struct_subfields)", "test_pushdown_struct_subfields", + ImmutableMap.of("x", toSubfields("x.b"))); + + assertPushdownSubfields("SELECT struct.b FROM (SELECT CUSTOM_STRUCT_WITHOUT_PASSTHROUGH(x) AS struct FROM test_pushdown_struct_subfields)", "test_pushdown_struct_subfields", + ImmutableMap.of()); + + assertPushdownSubfields("SELECT struct.b FROM (SELECT x AS struct FROM test_pushdown_struct_subfields)", "test_pushdown_struct_subfields", + ImmutableMap.of("x", toSubfields("x.b"))); + // Join assertPlan("SELECT l.orderkey, x.a, mod(x.d.d1, 2) FROM lineitem l, test_pushdown_struct_subfields a WHERE l.linenumber = a.id", anyTree( diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/annotations/FunctionsParserHelper.java b/presto-main-base/src/main/java/com/facebook/presto/operator/annotations/FunctionsParserHelper.java index acb63e7abe084..516fa0b58b258 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/annotations/FunctionsParserHelper.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/annotations/FunctionsParserHelper.java @@ -15,7 +15,9 @@ import com.facebook.presto.common.function.OperatorType; import com.facebook.presto.common.type.TypeSignature; +import com.facebook.presto.spi.function.ComplexTypeFunctionDescriptor; import com.facebook.presto.spi.function.Description; +import com.facebook.presto.spi.function.FunctionDescriptor; import com.facebook.presto.spi.function.IsNull; import com.facebook.presto.spi.function.LiteralParameters; import com.facebook.presto.spi.function.LongVariableConstraint; @@ -58,12 +60,16 @@ import static com.facebook.presto.common.function.OperatorType.NOT_EQUAL; import static com.facebook.presto.common.type.StandardTypes.PARAMETRIC_TYPES; import static com.facebook.presto.operator.annotations.ImplementationDependency.isImplementationDependencyAnnotation; +import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_ERROR; +import static com.facebook.presto.util.Failures.checkCondition; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static java.lang.reflect.Modifier.isPublic; import static java.lang.reflect.Modifier.isStatic; import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; public class FunctionsParserHelper { @@ -254,6 +260,30 @@ public static Optional parseDescription(AnnotatedElement base) return (description == null) ? Optional.empty() : Optional.of(description.value()); } + public static ComplexTypeFunctionDescriptor parseFunctionDescriptor(AnnotatedElement base) + { + FunctionDescriptor descriptor = base.getAnnotation(FunctionDescriptor.class); + if (descriptor == null) { + return ComplexTypeFunctionDescriptor.DEFAULT; + } + + int pushdownSubfieldArgIndex = descriptor.pushdownSubfieldArgIndex(); + Optional descriptorPushdownIndex; + if (pushdownSubfieldArgIndex < 0) { + descriptorPushdownIndex = Optional.empty(); + } + else { + descriptorPushdownIndex = Optional.of(pushdownSubfieldArgIndex); + } + + return new ComplexTypeFunctionDescriptor( + true, + emptyList(), + Optional.of(emptySet()), + Optional.of(ComplexTypeFunctionDescriptor::allSubfieldsRequired), + descriptorPushdownIndex); + } + public static List parseLongVariableConstraints(Method inputFunction) { return Stream.of(inputFunction.getAnnotationsByType(Constraint.class)) @@ -277,4 +307,25 @@ public static Map> getDeclaredSpecializedTypeParameters(Method } return specializedTypeParameters; } + + public static void checkPushdownSubfieldArgIndex(Method method, Signature signature, Optional pushdownSubfieldArgIndex) + { + if (pushdownSubfieldArgIndex.isPresent()) { + Map typeConstraintMapping = new HashMap<>(); + for (TypeVariableConstraint constraint : signature.getTypeVariableConstraints()) { + typeConstraintMapping.put(constraint.getName(), constraint); + } + checkCondition(signature.getArgumentTypes().size() > pushdownSubfieldArgIndex.get(), FUNCTION_IMPLEMENTATION_ERROR, "Method [%s] has out of range pushdown subfield arg index", method); + String typeVariableName = signature.getArgumentTypes().get(pushdownSubfieldArgIndex.get()).toString(); + + // The type variable must be directly a ROW type + // or (it is a type alias that is not bounded by a type) + // or (it is a type alias that maps to a row type) + boolean meetsTypeConstraint = (!typeConstraintMapping.containsKey(typeVariableName) && typeVariableName.equals(com.facebook.presto.common.type.StandardTypes.ROW)) || + (typeConstraintMapping.containsKey(typeVariableName) && typeConstraintMapping.get(typeVariableName).getVariadicBound() == null && !typeConstraintMapping.get(typeVariableName).isNonDecimalNumericRequired()) || + (typeConstraintMapping.containsKey(typeVariableName) && typeConstraintMapping.get(typeVariableName).getVariadicBound().equals(com.facebook.presto.common.type.StandardTypes.ROW)); + + checkCondition(meetsTypeConstraint, FUNCTION_IMPLEMENTATION_ERROR, "Method [%s] does not have a struct or row type as pushdown subfield arg", method); + } + } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/ParametricScalar.java b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/ParametricScalar.java index 3950c075e9fe4..f9ed7bef36118 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/ParametricScalar.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/ParametricScalar.java @@ -19,6 +19,7 @@ import com.facebook.presto.operator.ParametricImplementationsGroup; import com.facebook.presto.operator.scalar.annotations.ParametricScalarImplementation; import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.function.ComplexTypeFunctionDescriptor; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.google.common.annotations.VisibleForTesting; @@ -67,6 +68,12 @@ public boolean isCalledOnNullInput() return details.isCalledOnNullInput(); } + @Override + public ComplexTypeFunctionDescriptor getComplexTypeFunctionDescriptor() + { + return details.getComplexTypeFunctionDescriptor(); + } + @Override public String getDescription() { diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/ScalarHeader.java b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/ScalarHeader.java index b6b5e33fa6c00..bcecb71cb43f3 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/ScalarHeader.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/ScalarHeader.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.scalar; +import com.facebook.presto.spi.function.ComplexTypeFunctionDescriptor; import com.facebook.presto.spi.function.SqlFunctionVisibility; import java.util.Optional; @@ -23,13 +24,15 @@ public class ScalarHeader private final SqlFunctionVisibility visibility; private final boolean deterministic; private final boolean calledOnNullInput; + private final ComplexTypeFunctionDescriptor complexTypeFunctionDescriptor; - public ScalarHeader(Optional description, SqlFunctionVisibility visibility, boolean deterministic, boolean calledOnNullInput) + public ScalarHeader(Optional description, SqlFunctionVisibility visibility, boolean deterministic, boolean calledOnNullInput, ComplexTypeFunctionDescriptor complexTypeFunctionDescriptor) { this.description = description; this.visibility = visibility; this.deterministic = deterministic; this.calledOnNullInput = calledOnNullInput; + this.complexTypeFunctionDescriptor = complexTypeFunctionDescriptor; } public Optional getDescription() @@ -51,4 +54,9 @@ public boolean isCalledOnNullInput() { return calledOnNullInput; } + + public ComplexTypeFunctionDescriptor getComplexTypeFunctionDescriptor() + { + return complexTypeFunctionDescriptor; + } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/CodegenScalarFromAnnotationsParser.java b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/CodegenScalarFromAnnotationsParser.java index ae83d270b8308..8ce2bab423946 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/CodegenScalarFromAnnotationsParser.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/CodegenScalarFromAnnotationsParser.java @@ -25,6 +25,7 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.function.BlockPosition; import com.facebook.presto.spi.function.CodegenScalarFunction; +import com.facebook.presto.spi.function.ComplexTypeFunctionDescriptor; import com.facebook.presto.spi.function.Description; import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.IsNull; @@ -51,7 +52,9 @@ import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.metadata.SignatureBinder.applyBoundVariables; +import static com.facebook.presto.operator.annotations.FunctionsParserHelper.checkPushdownSubfieldArgIndex; import static com.facebook.presto.operator.annotations.FunctionsParserHelper.findPublicStaticMethods; +import static com.facebook.presto.operator.annotations.FunctionsParserHelper.parseFunctionDescriptor; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.functionTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; @@ -122,6 +125,8 @@ private static SqlScalarFunction createSqlScalarFunction(Method method) Arrays.stream(method.getParameters()).map(p -> parseTypeSignature(p.getAnnotation(SqlType.class).value())).collect(toImmutableList()), false); + ComplexTypeFunctionDescriptor descriptor = parseAndCheckFunctionDescriptor(method, signature); + return new SqlScalarFunction(signature) { @Override @@ -166,6 +171,19 @@ public boolean isCalledOnNullInput() { return codegenScalarFunction.calledOnNullInput(); } + + @Override + public ComplexTypeFunctionDescriptor getComplexTypeFunctionDescriptor() + { + return descriptor; + } }; } + + private static ComplexTypeFunctionDescriptor parseAndCheckFunctionDescriptor(Method method, Signature signature) + { + ComplexTypeFunctionDescriptor descriptor = parseFunctionDescriptor(method); + checkPushdownSubfieldArgIndex(method, signature, descriptor.getPushdownSubfieldArgIndex()); + return descriptor; + } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarFromAnnotationsParser.java b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarFromAnnotationsParser.java index eb2f5a3ae346f..bc5970d90e9b9 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarFromAnnotationsParser.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarFromAnnotationsParser.java @@ -19,6 +19,7 @@ import com.facebook.presto.operator.scalar.ParametricScalar; import com.facebook.presto.operator.scalar.annotations.ParametricScalarImplementation.SpecializedSignature; import com.facebook.presto.spi.function.CodegenScalarFunction; +import com.facebook.presto.spi.function.FunctionDescriptor; import com.facebook.presto.spi.function.ScalarFunction; import com.facebook.presto.spi.function.ScalarOperator; import com.facebook.presto.spi.function.Signature; @@ -35,6 +36,7 @@ import java.util.Optional; import java.util.Set; +import static com.facebook.presto.operator.annotations.FunctionsParserHelper.checkPushdownSubfieldArgIndex; import static com.facebook.presto.operator.scalar.annotations.OperatorValidator.validateOperator; import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_ERROR; import static com.facebook.presto.util.Failures.checkCondition; @@ -88,7 +90,7 @@ private static List findScalarsInFunctionSetClass(Class< ImmutableList.Builder builder = ImmutableList.builder(); for (Method method : FunctionsParserHelper.findPublicMethods( annotated, - ImmutableSet.of(SqlType.class, ScalarFunction.class, ScalarOperator.class), + ImmutableSet.of(SqlType.class, ScalarFunction.class, ScalarOperator.class, FunctionDescriptor.class), ImmutableSet.of(SqlInvokedScalarFunction.class, CodegenScalarFunction.class))) { checkCondition((method.getAnnotation(ScalarFunction.class) != null) || (method.getAnnotation(ScalarOperator.class) != null), FUNCTION_IMPLEMENTATION_ERROR, "Method [%s] annotated with @SqlType is missing @ScalarFunction or @ScalarOperator", method); @@ -106,6 +108,7 @@ private static SqlScalarFunction parseParametricScalar(ScalarHeaderAndMethods sc Map signatures = new HashMap<>(); for (Method method : scalar.getMethods()) { ParametricScalarImplementation implementation = ParametricScalarImplementation.Parser.parseImplementation(header, method, constructor); + checkPushdownSubfieldArgIndex(method, implementation.getSignature(), header.getHeader().getComplexTypeFunctionDescriptor().getPushdownSubfieldArgIndex()); if (!signatures.containsKey(implementation.getSpecializedSignature())) { ParametricScalarImplementation.Builder builder = new ParametricScalarImplementation.Builder( implementation.getSignature(), diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarImplementationHeader.java b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarImplementationHeader.java index b79b0b7f05219..2a3045bed71b1 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarImplementationHeader.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarImplementationHeader.java @@ -16,6 +16,7 @@ import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.function.OperatorType; import com.facebook.presto.operator.scalar.ScalarHeader; +import com.facebook.presto.spi.function.ComplexTypeFunctionDescriptor; import com.facebook.presto.spi.function.ScalarFunction; import com.facebook.presto.spi.function.ScalarOperator; import com.facebook.presto.spi.function.SqlFunctionVisibility; @@ -28,6 +29,7 @@ import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.annotations.FunctionsParserHelper.parseDescription; +import static com.facebook.presto.operator.annotations.FunctionsParserHelper.parseFunctionDescriptor; import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; import static com.google.common.base.CaseFormat.LOWER_CAMEL; import static com.google.common.base.CaseFormat.LOWER_UNDERSCORE; @@ -76,20 +78,21 @@ public static List fromAnnotatedElement(AnnotatedEle ScalarFunction scalarFunction = annotated.getAnnotation(ScalarFunction.class); ScalarOperator scalarOperator = annotated.getAnnotation(ScalarOperator.class); Optional description = parseDescription(annotated); + ComplexTypeFunctionDescriptor descriptor = parseFunctionDescriptor(annotated); ImmutableList.Builder builder = ImmutableList.builder(); if (scalarFunction != null) { String baseName = scalarFunction.value().isEmpty() ? camelToSnake(annotatedName(annotated)) : scalarFunction.value(); - builder.add(new ScalarImplementationHeader(baseName, new ScalarHeader(description, scalarFunction.visibility(), scalarFunction.deterministic(), scalarFunction.calledOnNullInput()))); + builder.add(new ScalarImplementationHeader(baseName, new ScalarHeader(description, scalarFunction.visibility(), scalarFunction.deterministic(), scalarFunction.calledOnNullInput(), descriptor))); for (String alias : scalarFunction.alias()) { - builder.add(new ScalarImplementationHeader(alias, new ScalarHeader(description, scalarFunction.visibility(), scalarFunction.deterministic(), scalarFunction.calledOnNullInput()))); + builder.add(new ScalarImplementationHeader(alias, new ScalarHeader(description, scalarFunction.visibility(), scalarFunction.deterministic(), scalarFunction.calledOnNullInput(), descriptor))); } } if (scalarOperator != null) { - builder.add(new ScalarImplementationHeader(scalarOperator.value(), new ScalarHeader(description, HIDDEN, true, scalarOperator.value().isCalledOnNullInput()))); + builder.add(new ScalarImplementationHeader(scalarOperator.value(), new ScalarHeader(description, HIDDEN, true, scalarOperator.value().isCalledOnNullInput(), descriptor))); } List result = builder.build(); diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java index 6a0a5d37c79d1..0334f8833390d 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java @@ -582,6 +582,16 @@ private static Optional toSubfield( if (expression instanceof VariableReferenceExpression) { return Optional.of(new Subfield(((VariableReferenceExpression) expression).getName(), elements.build().reverse())); } + if (expression instanceof CallExpression) { + ComplexTypeFunctionDescriptor functionDescriptor = functionAndTypeManager.getFunctionMetadata(((CallExpression) expression).getFunctionHandle()).getDescriptor(); + Optional pushdownSubfieldArgIndex = functionDescriptor.getPushdownSubfieldArgIndex(); + if (pushdownSubfieldArgIndex.isPresent() && + ((CallExpression) expression).getArguments().size() > pushdownSubfieldArgIndex.get() && + ((CallExpression) expression).getArguments().get(pushdownSubfieldArgIndex.get()).getType() instanceof RowType) { + expression = ((CallExpression) expression).getArguments().get(pushdownSubfieldArgIndex.get()); + continue; + } + } if (expression instanceof SpecialFormExpression && ((SpecialFormExpression) expression).getForm() == DEREFERENCE) { SpecialFormExpression dereference = (SpecialFormExpression) expression; diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/ComplexTypeFunctionDescriptor.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/ComplexTypeFunctionDescriptor.java index a2d5c1db531bf..005af4ae9e296 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/ComplexTypeFunctionDescriptor.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/ComplexTypeFunctionDescriptor.java @@ -44,7 +44,8 @@ public class ComplexTypeFunctionDescriptor true, emptyList(), Optional.of(emptySet()), - Optional.of(ComplexTypeFunctionDescriptor::allSubfieldsRequired)); + Optional.of(ComplexTypeFunctionDescriptor::allSubfieldsRequired), + Optional.empty()); /** * Indicates whether the function accessing subfields. @@ -74,6 +75,12 @@ public class ComplexTypeFunctionDescriptor */ private final List lambdaDescriptors; + /** + * pushdownSubfieldArgIndex is used to specify which parameter in the scalar function corresponds to the parameter that should have its subfields pushed down to filter plan nodes during query planning and optimization + * This is helpful to reduce the amount of scanning done for queries involving structs, and it ensures that only utilized subfields of the struct scanned and unused subfields can be pruned from the query plan + */ + private final Optional pushdownSubfieldArgIndex; + public ComplexTypeFunctionDescriptor( boolean isAccessingInputValues, List lambdaDescriptors, @@ -83,6 +90,7 @@ public ComplexTypeFunctionDescriptor( { this(isAccessingInputValues, lambdaDescriptors, argumentIndicesContainingMapOrArray, outputToInputTransformationFunction, signature.getArgumentTypes()); } + public ComplexTypeFunctionDescriptor( boolean isAccessingInputValues, List lambdaDescriptors, @@ -90,7 +98,18 @@ public ComplexTypeFunctionDescriptor( Optional, Set>> outputToInputTransformationFunction, List argumentTypes) { - this(isAccessingInputValues, lambdaDescriptors, argumentIndicesContainingMapOrArray, outputToInputTransformationFunction); + this(isAccessingInputValues, lambdaDescriptors, argumentIndicesContainingMapOrArray, outputToInputTransformationFunction, argumentTypes, Optional.empty()); + } + + public ComplexTypeFunctionDescriptor( + boolean isAccessingInputValues, + List lambdaDescriptors, + Optional> argumentIndicesContainingMapOrArray, + Optional, Set>> outputToInputTransformationFunction, + List argumentTypes, + Optional pushdownSubfieldArgIndex) + { + this(isAccessingInputValues, lambdaDescriptors, argumentIndicesContainingMapOrArray, outputToInputTransformationFunction, pushdownSubfieldArgIndex); if (argumentIndicesContainingMapOrArray.isPresent()) { checkArgument(argumentIndicesContainingMapOrArray.get().stream().allMatch(index -> index >= 0 && index < argumentTypes.size() && @@ -115,6 +134,16 @@ public ComplexTypeFunctionDescriptor( List lambdaDescriptors, Optional> argumentIndicesContainingMapOrArray, Optional, Set>> outputToInputTransformationFunction) + { + this(isAccessingInputValues, lambdaDescriptors, argumentIndicesContainingMapOrArray, outputToInputTransformationFunction, Optional.empty()); + } + + public ComplexTypeFunctionDescriptor( + boolean isAccessingInputValues, + List lambdaDescriptors, + Optional> argumentIndicesContainingMapOrArray, + Optional, Set>> outputToInputTransformationFunction, + Optional pushdownSubfieldArgIndex) { requireNonNull(argumentIndicesContainingMapOrArray, "argumentIndicesContainingMapOrArray is null"); this.isAccessingInputValues = isAccessingInputValues; @@ -123,6 +152,7 @@ public ComplexTypeFunctionDescriptor( Optional.of(unmodifiableSet(argumentIndicesContainingMapOrArray.get())) : Optional.empty(); this.outputToInputTransformationFunction = requireNonNull(outputToInputTransformationFunction, "outputToInputTransformationFunction is null"); + this.pushdownSubfieldArgIndex = requireNonNull(pushdownSubfieldArgIndex, "pushdownSubfieldArgIndex is null"); } public static ComplexTypeFunctionDescriptor defaultFunctionDescriptor() @@ -155,6 +185,11 @@ public Optional, Set>> getOutputToInputTransfor return outputToInputTransformationFunction; } + public Optional getPushdownSubfieldArgIndex() + { + return pushdownSubfieldArgIndex; + } + @Override public boolean equals(Object o) { diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionDescriptor.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionDescriptor.java new file mode 100644 index 0000000000000..d0d312b3533f3 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionDescriptor.java @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.function; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Retention(RUNTIME) +@Target({METHOD, PARAMETER}) +public @interface FunctionDescriptor +{ + int pushdownSubfieldArgIndex() default -1; +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadata.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadata.java index a82a744b32fbe..995068247ec99 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadata.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadata.java @@ -65,7 +65,19 @@ public FunctionMetadata( boolean calledOnNullInput, ComplexTypeFunctionDescriptor functionDescriptor) { - this(name, Optional.empty(), argumentTypes, Optional.empty(), returnType, functionKind, Optional.empty(), implementationType, deterministic, calledOnNullInput, notVersioned(), functionDescriptor); + this( + name, + Optional.empty(), + argumentTypes, + Optional.empty(), + returnType, + functionKind, + Optional.empty(), + implementationType, + deterministic, + calledOnNullInput, + notVersioned(), + functionDescriptor); } public FunctionMetadata( @@ -184,8 +196,10 @@ private FunctionMetadata( functionDescriptor.getLambdaDescriptors(), functionDescriptor.getArgumentIndicesContainingMapOrArray(), functionDescriptor.getOutputToInputTransformationFunction(), - argumentTypes); + argumentTypes, + functionDescriptor.getPushdownSubfieldArgIndex()); } + public FunctionKind getFunctionKind() { return functionKind; diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java index ad37f43f6a0c4..755c45262fc2e 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java @@ -145,6 +145,8 @@ public abstract class AbstractTestQueries .window(CustomRank.class) .scalars(CustomAdd.class) .scalars(CreateHll.class) + .scalars(CustomStructWithPassthrough.class) + .scalars(CustomStructWithoutPassthrough.class) .functions(APPLY_FUNCTION, INVOKE_FUNCTION, STATEFUL_SLEEPING_SUM) .getFunctions(); diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/CustomStructWithPassthrough.java b/presto-tests/src/main/java/com/facebook/presto/tests/CustomStructWithPassthrough.java new file mode 100644 index 0000000000000..28803efa727a5 --- /dev/null +++ b/presto-tests/src/main/java/com/facebook/presto/tests/CustomStructWithPassthrough.java @@ -0,0 +1,36 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.tests; + +import com.facebook.presto.common.block.Block; +import com.facebook.presto.spi.function.FunctionDescriptor; +import com.facebook.presto.spi.function.ScalarFunction; +import com.facebook.presto.spi.function.SqlType; +import com.facebook.presto.spi.function.TypeParameter; + +import static com.facebook.presto.common.type.StandardTypes.ROW; + +public final class CustomStructWithPassthrough +{ + private CustomStructWithPassthrough() {} + + @ScalarFunction(value = "custom_struct_with_passthrough") + @TypeParameter(value = "T", boundedBy = ROW) + @FunctionDescriptor(pushdownSubfieldArgIndex = 0) + @SqlType("T") + public static Block customStructWithPassthrough(@SqlType("T") Block struct) + { + return struct; + } +} diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/CustomStructWithoutPassthrough.java b/presto-tests/src/main/java/com/facebook/presto/tests/CustomStructWithoutPassthrough.java new file mode 100644 index 0000000000000..854ca795269d9 --- /dev/null +++ b/presto-tests/src/main/java/com/facebook/presto/tests/CustomStructWithoutPassthrough.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.tests; + +import com.facebook.presto.common.block.Block; +import com.facebook.presto.spi.function.ScalarFunction; +import com.facebook.presto.spi.function.SqlType; +import com.facebook.presto.spi.function.TypeParameter; + +import static com.facebook.presto.common.type.StandardTypes.ROW; + +public final class CustomStructWithoutPassthrough +{ + private CustomStructWithoutPassthrough() {} + + @ScalarFunction(value = "custom_struct_without_passthrough") + @TypeParameter(value = "T", boundedBy = ROW) + @SqlType("T") + public static Block customStructWithoutPassthrough(@SqlType("T") Block struct) + { + return struct; + } +}