diff --git a/.travis.yml b/.travis.yml index 71ba451a0ece3..b68bd71770e68 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,27 +29,43 @@ dist: trusty cache: directories: - $HOME/.m2/repository + - $HOME/.thrift services: - docker +before_install: + - | + if [[ ! -e $HOME/.thrift/bin/thrift ]]; then + sudo apt-get install libboost-dev libboost-test-dev libboost-program-options-dev libboost-filesystem-dev libboost-thread-dev libevent-dev automake libtool flex bison pkg-config g++ libssl-dev + wget https://www.apache.org/dist/thrift/0.9.3/thrift-0.9.3.tar.gz + tar xfz thrift-0.9.3.tar.gz + cd thrift-0.9.3 && ./configure --without-cpp --without-c_glib --without-python --without-ruby --without-php --without-erlang --without-go --without-nodejs -q --prefix=$HOME/.thrift + sudo make install > thrift_make_install.log + cd .. + fi + - | + if [[ ! -e /usr/local/bin/thrift ]]; then + sudo ln -s $HOME/.thrift/bin/thrift /usr/local/bin/thrift + fi + install: - ./mvnw -v - | if [[ -v TEST_SPECIFIC_MODULES ]]; then - ./mvnw install $MAVEN_FAST_INSTALL -pl $TEST_SPECIFIC_MODULES -am + ./mvnw install $MAVEN_FAST_INSTALL -P !twitter-modules -pl $TEST_SPECIFIC_MODULES -am fi - | if [[ -v TEST_OTHER_MODULES ]]; then - ./mvnw install $MAVEN_FAST_INSTALL -pl '!presto-docs,!presto-server,!presto-server-rpm' + ./mvnw install $MAVEN_FAST_INSTALL -P !twitter-modules -pl '!presto-docs,!presto-server,!presto-server-rpm' fi - | if [[ -v PRODUCT_TESTS_BASIC_ENVIRONMENT || -v PRODUCT_TESTS_SPECIFIC_ENVIRONMENT || -v PRODUCT_TESTS_SPECIFIC_ENVIRONMENT_2 ]]; then - ./mvnw install $MAVEN_FAST_INSTALL -pl '!presto-docs,!presto-server-rpm' + ./mvnw install $MAVEN_FAST_INSTALL -P !twitter-modules -pl '!presto-docs,!presto-server-rpm' fi - | if [[ -v HIVE_TESTS ]]; then - ./mvnw install $MAVEN_FAST_INSTALL -pl presto-hive-hadoop2 -am + ./mvnw install $MAVEN_FAST_INSTALL -P !twitter-modules -pl presto-hive-hadoop2 -am fi before_script: @@ -71,15 +87,15 @@ before_script: script: - | if [[ -v MAVEN_CHECKS ]]; then - ./mvnw install -DskipTests -B -T C1 -P ci + ./mvnw install -DskipTests -B -T C1 -P 'ci,!twitter-modules' fi - | if [[ -v TEST_SPECIFIC_MODULES ]]; then - ./mvnw test $MAVEN_SKIP_CHECKS_AND_DOCS -B -pl $TEST_SPECIFIC_MODULES $TEST_FLAGS + ./mvnw test $MAVEN_SKIP_CHECKS_AND_DOCS -P !twitter-modules -B -pl $TEST_SPECIFIC_MODULES $TEST_FLAGS fi - | if [[ -v TEST_OTHER_MODULES ]]; then - ./mvnw test $MAVEN_SKIP_CHECKS_AND_DOCS -B -pl $TEST_OTHER_MODULES + ./mvnw test $MAVEN_SKIP_CHECKS_AND_DOCS -P !twitter-modules -B -pl $TEST_OTHER_MODULES fi - | if [[ -v PRODUCT_TESTS_BASIC_ENVIRONMENT ]]; then @@ -122,7 +138,7 @@ script: - | if [[ -v PRODUCT_TESTS_SPECIFIC_ENVIRONMENT_2 ]]; then presto-product-tests/bin/run_on_docker.sh \ - multinode-tls -g smoke,cli,group-by,join,tls + multinode-tls -g smoke,group-by,join,tls fi - | if [[ -v PRODUCT_TESTS_SPECIFIC_ENVIRONMENT_2 ]]; then @@ -177,8 +193,11 @@ before_cache: - rm -rf $HOME/.m2/repository/com/facebook notifications: + hipchat: + rooms: + secure: peNh1KxwlxIpFyb60S8AMvaJThgh1LsjE+Whf1rYkJalVd2wUrqBIoyDKVSueyHD01hQ06gT7rBV6Pu/QcBMR1a9BbMCjERfxLZFUAheuC2Rsb+p1c4dyvBcFUGacgW7XWKCaVYGDGxuUvb0I3Z8cR6KxhK2xi88tHiqBGVGV2yI6zzOTpWVknMfFBtn+ONU1Ob2P6trclXaDyFd4MxubULri6CQdl35eQAq/VnmR3SZOgyVu3V30MGKwI3zhSli+3VqmW0JmaDGoHN6gznM1+VqABLgmIq0P+n+r5gdZWRCorq10NZCFMhVQ8U6rQHcL7sAniYJJsC/yRt6+pjyzIF4N+LSzZ7T+FLxQqT7k/1ukNgrujLDfTpn76Mo9eYTZmfAdzbm1QKJDACwr8Slqhq1jGzcrFMHunvXhVqjOs24R+JAHblY0O9PXvv7aR29GOQWDCvD7nV5QBUr8Xz5q7ozbLqHTI+yH02Jj4EaZ+azWYdRmnr9wDBxWMYBEgOdj4pII9b298XEDB72TxA3KpLTpdLxBTR+gIk/LjJqb/wb84xUv8gPXkaXccltGd5YI90c84cX8isbzNkAylzyfF2Eyueh0XbnMHfpFqBS7qaVM0/D+UxZkU0WNJ0x7G9XJvkiq49bZz2q1KLE4XuvVnTZSSjVSUAS8RtHfwUV33c= slack: - secure: V5eyoGShxFoCcYJcp858vf/T6gC9KeMxL0C1EElcpZRcKBrIVZzvhek3HLHxZOxlghqnvNVsyDtU3u5orkEaAXeXj5c2dN+4XBsAB9oeN5MtQ0Z3VLAhZDqKIW1LzcXrq4DpzM0PkGhjfjum/P94/qFYk0UckPtB6a341AuYRo8= + secure: E7XVlbdwIdKxnr6Tk1rmCefufs1w8h4nCWz79Uh6wMma8gC7x5ChKFqwvLRJ0WUpmPS+Ng1xeTv+wmb8TMDv2X8snmht9420/TFRy9wi1aLWNJXQUveNBzn83sCS40jFi6gd9xqKawd68R84UVH3PeNhksDtDnKAblx71miwbKmLwHc1KFoLMEnaaWEg5NgFl8/UadYDvsLD44v6YDza8eYrLp3aGK8v9ewBDySHE16IHAfpteTRaU0kG/H1kvVvFdH/h/sSPfimehd51b4i3mm/nRrjJ/VSLc7p9w5FkHUECtA0N6zcytRxN6MrbhrxJ8XG3vte3KSRSFCqfgOSRM2NWcca4CtBP2V0SwrAYMo5jim6fr921lfcbUTWTSnvMYLC17QrAxoclVrgK05GjGoLgSH42UPGf3QNkqXzyueNzaLJ+KSlgwFblIQKp6WGZYSRorL0F7s50pIoqMVoebcrnB0ObK/CcE2ywS/HeTgoSkWSDSmKBsO+cmtv1yAamy9DlmgRGZlxIxdBELXtHRkQ2B6Z2QdiQU4MHiFBc/IESJbnCait4odn+oJUjehZg+b9vjCoWwVw3zNMIJhokyxO8SiyKJmbO0z1g2L/BykWGI1DQu8HkeQzO+CmNUV3AOrxDG3amL/tkB/06fyQtnYMDhUhvX64uWSaE36sYL4= before_deploy: - mkdir /tmp/artifacts diff --git a/README.md b/README.md index 800bfddc21c0f..8b572bf54111f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Presto [![Build Status](https://travis-ci.org/prestodb/presto.svg?branch=master)](https://travis-ci.org/prestodb/presto) +# Presto [![Build Status](https://travis-ci.org/twitter-forks/presto.svg?branch=twitter-master)](https://travis-ci.org/twitter-forks/presto) Presto is a distributed SQL query engine for big data. diff --git a/pom.xml b/pom.xml index 987a85225568c..ffbaac06cfafb 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 pom presto-root @@ -28,10 +28,10 @@ - scm:git:git://github.com/facebook/presto.git - https://github.com/facebook/presto - 0.208 - + scm:git:git://github.com/twitter-forks/presto.git + https://github.com/twitter-forks/presto + 0.208-tw-0.53 + ${project.basedir} @@ -117,6 +117,7 @@ presto-thrift-testing-server presto-thrift-connector presto-matching + presto-twitter-functions presto-memory-context presto-proxy @@ -801,6 +802,12 @@ 3.6.1 + + org.apache.commons + commons-pool2 + 2.4.2 + + io.airlift.discovery discovery-server @@ -889,6 +896,54 @@ 1.1.2.6 + + org.apache.curator + curator-recipes + 4.0.0 + + + org.apache.zookeeper + zookeeper + + + + + + org.apache.curator + curator-framework + 4.0.0 + + + org.apache.zookeeper + zookeeper + + + + + + org.apache.curator + curator-client + 4.0.0 + + + org.apache.zookeeper + zookeeper + + + + + + org.apache.curator + curator-test + 2.12.0 + + + org.apache.zookeeper + zookeeper + + + + org.apache.zookeeper zookeeper @@ -909,6 +964,26 @@ + + com.101tec + zkclient + 0.10 + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + org.apache.zookeeper + zookeeper + + + + org.jgrapht jgrapht-core @@ -1224,6 +1299,17 @@ + + twitter-modules + + true + + + presto-kafka07 + twitter-eventlistener-plugin + presto-twitter-server + + tests-with-dependencies io.airlift @@ -204,6 +265,18 @@ + + org.apache.curator + curator-test + test + + + + com.101tec + zkclient + test + + com.facebook.presto presto-main @@ -252,12 +325,6 @@ test - - org.anarres.lzo - lzo-hadoop - test - - com.facebook.presto diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java index 895a5e53c1a0c..ef94a1e238ff4 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java @@ -100,6 +100,7 @@ public class HiveClientConfig private boolean useParquetColumnNames; private boolean parquetOptimizedReaderEnabled = true; private boolean parquetPredicatePushdownEnabled = true; + private boolean parquetNestedFieldsProjectionPushdownEnabled; private boolean assumeCanonicalPartitionKeys; @@ -679,6 +680,18 @@ public HiveClientConfig setParquetPredicatePushdownEnabled(boolean parquetPredic return this; } + public boolean isParquetNestedFieldsProjectionPushdownEnabled() + { + return parquetNestedFieldsProjectionPushdownEnabled; + } + + @Config("hive.parquet-nested-fields-projection-pushdown.enabled") + public HiveClientConfig setParquetNestedFieldsProjectionPushdownEnabled(boolean parquetNestedFieldsProjectionPushdownEnabled) + { + this.parquetNestedFieldsProjectionPushdownEnabled = parquetNestedFieldsProjectionPushdownEnabled; + return this; + } + @Deprecated public boolean isParquetOptimizedReaderEnabled() { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java index b0bb04cb410a3..a9f7b8aa74c7b 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java @@ -23,6 +23,9 @@ import com.facebook.presto.spi.connector.ConnectorPageSinkProvider; import com.facebook.presto.spi.connector.ConnectorPageSourceProvider; import com.facebook.presto.spi.connector.ConnectorSplitManager; +import com.facebook.presto.twitter.hive.thrift.HiveThriftFieldIdResolverFactory; +import com.facebook.presto.twitter.hive.thrift.ThriftFieldIdResolverFactory; +import com.facebook.presto.twitter.hive.thrift.ThriftHiveRecordCursorProvider; import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Provides; @@ -72,9 +75,11 @@ public void configure(Binder binder) binder.bind(NamenodeStats.class).in(Scopes.SINGLETON); newExporter(binder).export(NamenodeStats.class).as(generatedNameOf(NamenodeStats.class, connectorId)); + binder.bind(ThriftFieldIdResolverFactory.class).toInstance(new HiveThriftFieldIdResolverFactory()); Multibinder recordCursorProviderBinder = newSetBinder(binder, HiveRecordCursorProvider.class); recordCursorProviderBinder.addBinding().to(ParquetRecordCursorProvider.class).in(Scopes.SINGLETON); + recordCursorProviderBinder.addBinding().to(ThriftHiveRecordCursorProvider.class).in(Scopes.SINGLETON); recordCursorProviderBinder.addBinding().to(GenericHiveRecordCursorProvider.class).in(Scopes.SINGLETON); binder.bind(HiveWriterStats.class).in(Scopes.SINGLETON); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveColumnHandle.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveColumnHandle.java index 0b153e7f71814..1d30bfc1d8f85 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveColumnHandle.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveColumnHandle.java @@ -19,9 +19,11 @@ import com.facebook.presto.spi.type.TypeSignature; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableSet; import java.util.Objects; import java.util.Optional; +import java.util.Set; import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY; import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.SYNTHESIZED; @@ -29,7 +31,6 @@ import static com.facebook.presto.hive.HiveType.HIVE_LONG; import static com.facebook.presto.hive.HiveType.HIVE_STRING; import static com.facebook.presto.spi.type.BigintType.BIGINT; -import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; @@ -56,15 +57,22 @@ public enum ColumnType } private final String name; + private final Optional> fieldSet; private final HiveType hiveType; private final TypeSignature typeName; private final int hiveColumnIndex; private final ColumnType columnType; private final Optional comment; + public HiveColumnHandle(String name, HiveType hiveType, TypeSignature typeSignature, int hiveColumnIndex, ColumnType columnType, Optional comment) + { + this(name, Optional.empty(), hiveType, typeSignature, hiveColumnIndex, columnType, comment); + } + @JsonCreator public HiveColumnHandle( @JsonProperty("name") String name, + @JsonProperty("fieldSet") Optional> fieldSet, @JsonProperty("hiveType") HiveType hiveType, @JsonProperty("typeSignature") TypeSignature typeSignature, @JsonProperty("hiveColumnIndex") int hiveColumnIndex, @@ -72,6 +80,7 @@ public HiveColumnHandle( @JsonProperty("comment") Optional comment) { this.name = requireNonNull(name, "name is null"); + this.fieldSet = requireNonNull(fieldSet, "fieldSet is null"); checkArgument(hiveColumnIndex >= 0 || columnType == PARTITION_KEY || columnType == SYNTHESIZED, "hiveColumnIndex is negative"); this.hiveColumnIndex = hiveColumnIndex; this.hiveType = requireNonNull(hiveType, "hiveType is null"); @@ -86,6 +95,12 @@ public String getName() return name; } + @JsonProperty + public Optional> getFieldSet() + { + return fieldSet; + } + @JsonProperty public HiveType getHiveType() { @@ -134,7 +149,7 @@ public ColumnType getColumnType() @Override public int hashCode() { - return Objects.hash(name, hiveColumnIndex, hiveType, columnType, comment); + return Objects.hash(name, fieldSet, hiveColumnIndex, hiveType, columnType, comment); } @Override @@ -148,6 +163,7 @@ public boolean equals(Object obj) } HiveColumnHandle other = (HiveColumnHandle) obj; return Objects.equals(this.name, other.name) && + Objects.equals(this.fieldSet, other.fieldSet) && Objects.equals(this.hiveColumnIndex, other.hiveColumnIndex) && Objects.equals(this.hiveType, other.hiveType) && Objects.equals(this.columnType, other.columnType) && @@ -157,14 +173,7 @@ public boolean equals(Object obj) @Override public String toString() { - return toStringHelper(this) - .add("name", name) - .add("hiveType", hiveType) - .add("hiveColumnIndex", hiveColumnIndex) - .add("columnType", columnType) - .add("comment", comment.orElse(null)) - .omitNullValues() - .toString(); + return name + ":" + fieldSet.orElse(ImmutableSet.of()) + ":" + hiveType + ":" + hiveColumnIndex + ":" + columnType; } public static HiveColumnHandle updateRowIdHandle() @@ -202,4 +211,9 @@ public static boolean isBucketColumnHandle(HiveColumnHandle column) { return column.getHiveColumnIndex() == BUCKET_COLUMN_INDEX; } + + public static HiveColumnHandle withFieldSet(HiveColumnHandle column, Optional> fieldSet) + { + return new HiveColumnHandle(column.name, fieldSet, column.getHiveType(), column.getTypeSignature(), column.getHiveColumnIndex(), column.getColumnType(), column.getComment()); + } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java index d41568db76162..e3223e3942ce6 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java @@ -113,6 +113,7 @@ import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.SYNTHESIZED; import static com.facebook.presto.hive.HiveColumnHandle.PATH_COLUMN_NAME; import static com.facebook.presto.hive.HiveColumnHandle.updateRowIdHandle; +import static com.facebook.presto.hive.HiveColumnHandle.withFieldSet; import static com.facebook.presto.hive.HiveErrorCode.HIVE_COLUMN_ORDER_MISMATCH; import static com.facebook.presto.hive.HiveErrorCode.HIVE_CONCURRENT_MODIFICATION_DETECTED; import static com.facebook.presto.hive.HiveErrorCode.HIVE_EXCEEDED_PARTITION_LIMIT; @@ -1398,20 +1399,44 @@ public List getTableLayouts(ConnectorSession session HiveTableHandle handle = (HiveTableHandle) tableHandle; HivePartitionResult hivePartitionResult = partitionManager.getPartitions(metastore, tableHandle, constraint); + HiveTableLayoutHandle layoutHandle = new HiveTableLayoutHandle( + handle.getSchemaTableName(), + ImmutableList.copyOf(hivePartitionResult.getPartitionColumns()), + getPartitionsAsList(hivePartitionResult), + hivePartitionResult.getCompactEffectivePredicate(), + hivePartitionResult.getEnforcedConstraint(), + hivePartitionResult.getBucketHandle(), + hivePartitionResult.getBucketFilter()); + + if (constraint.fieldSets().isPresent()) { + return ImmutableList.of(new ConnectorTableLayoutResult( + pruneColumnFields(layoutHandle, constraint), + constraint.getSummary())); + } + return ImmutableList.of(new ConnectorTableLayoutResult( - getTableLayout( - session, - new HiveTableLayoutHandle( - handle.getSchemaTableName(), - ImmutableList.copyOf(hivePartitionResult.getPartitionColumns()), - getPartitionsAsList(hivePartitionResult), - hivePartitionResult.getCompactEffectivePredicate(), - hivePartitionResult.getEnforcedConstraint(), - hivePartitionResult.getBucketHandle(), - hivePartitionResult.getBucketFilter())), + getTableLayout(session, layoutHandle), hivePartitionResult.getUnenforcedConstraint())); } + private ConnectorTableLayout pruneColumnFields(HiveTableLayoutHandle layoutHandle, Constraint constraint) + { + Optional> columns = constraint.fieldSets() + .map(fieldsSets -> fieldsSets.stream() + .filter(fieldSet -> !((HiveColumnHandle) fieldSet.getColumn()).getFieldSet().isPresent()) + .map(fieldSet -> withFieldSet((HiveColumnHandle) fieldSet.getColumn(), Optional.of(fieldSet.getFields()))) + .collect(toImmutableList())); + + return new ConnectorTableLayout( + layoutHandle, + columns, + TupleDomain.all(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + emptyList()); + } + @Override public ConnectorTableLayout getTableLayout(ConnectorSession session, ConnectorTableLayoutHandle layoutHandle) { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java index a8ea96480e6d9..6d2bc4b37382c 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HivePageSourceProvider.java @@ -356,6 +356,7 @@ public static List toColumnHandles(List regular } return new HiveColumnHandle( columnHandle.getName(), + columnHandle.getFieldSet(), columnMapping.getCoercionFrom().get(), columnMapping.getCoercionFrom().get().getTypeSignature(), columnHandle.getHiveColumnIndex(), diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java index 62988901396a5..0490ee2611fa3 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveSessionProperties.java @@ -64,6 +64,7 @@ public final class HiveSessionProperties private static final String PARQUET_PREDICATE_PUSHDOWN_ENABLED = "parquet_predicate_pushdown_enabled"; private static final String PARQUET_OPTIMIZED_READER_ENABLED = "parquet_optimized_reader_enabled"; private static final String PARQUET_USE_COLUMN_NAME = "parquet_use_column_names"; + private static final String PARQUET_NESTED_FIELDS_PROJECTION_PUSHDOWN_READER_ENABLED = "parquet_nested_fields_projection_pushdown_enabled"; private static final String MAX_SPLIT_SIZE = "max_split_size"; private static final String MAX_INITIAL_SPLIT_SIZE = "max_initial_split_size"; public static final String RCFILE_OPTIMIZED_WRITER_ENABLED = "rcfile_optimized_writer_enabled"; @@ -221,6 +222,11 @@ public HiveSessionProperties(HiveClientConfig hiveClientConfig, OrcFileWriterCon "Experimental: Parquet: Access Parquet columns using names from the file", hiveClientConfig.isUseParquetColumnNames(), false), + booleanProperty( + PARQUET_NESTED_FIELDS_PROJECTION_PUSHDOWN_READER_ENABLED, + "Experimental: Parquet: Enable nested fields projection pushdown for Parquet", + hiveClientConfig.isParquetNestedFieldsProjectionPushdownEnabled(), + false), dataSizeSessionProperty( MAX_SPLIT_SIZE, "Max split size", @@ -398,6 +404,11 @@ public static boolean isUseParquetColumnNames(ConnectorSession session) return session.getProperty(PARQUET_USE_COLUMN_NAME, Boolean.class); } + public static boolean isParquetNestedFieldsProjectionPushdownEnabled(ConnectorSession session) + { + return session.getProperty(PARQUET_NESTED_FIELDS_PROJECTION_PUSHDOWN_READER_ENABLED, Boolean.class); + } + public static DataSize getMaxSplitSize(ConnectorSession session) { return session.getProperty(MAX_SPLIT_SIZE, DataSize.class); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveStorageFormat.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveStorageFormat.java index 4e4523d041859..5a9edf5b383ce 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveStorageFormat.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveStorageFormat.java @@ -14,6 +14,7 @@ package com.facebook.presto.hive; import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.twitter.hive.thrift.ThriftGeneralInputFormat; import io.airlift.units.DataSize; import io.airlift.units.DataSize.Unit; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; @@ -32,6 +33,7 @@ import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; import org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; +import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; @@ -93,6 +95,11 @@ public enum HiveStorageFormat LazySimpleSerDe.class.getName(), TextInputFormat.class.getName(), HiveIgnoreKeyTextOutputFormat.class.getName(), + new DataSize(8, Unit.MEGABYTE)), + THRIFTBINARY( + LazyBinarySerDe.class.getName(), + ThriftGeneralInputFormat.class.getName(), + HiveIgnoreKeyTextOutputFormat.class.getName(), new DataSize(8, Unit.MEGABYTE)); private final String serde; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java index 8c774ebd8cb14..9c895eeeeeba9 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java @@ -28,6 +28,7 @@ import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.VarcharType; +import com.facebook.presto.twitter.hive.thrift.ThriftGeneralInputFormat; import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.base.VerifyException; @@ -40,6 +41,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.SymlinkTextInputFormat; @@ -119,6 +121,7 @@ import static com.google.common.collect.Iterables.filter; import static com.google.common.collect.Lists.newArrayList; import static com.google.common.collect.Lists.transform; +import static com.hadoop.compression.lzo.LzoIndex.LZO_INDEX_SUFFIX; import static java.lang.Byte.parseByte; import static java.lang.Double.parseDouble; import static java.lang.Float.floatToRawIntBits; @@ -155,6 +158,22 @@ public final class HiveUtil private static final String BIG_DECIMAL_POSTFIX = "BD"; + private static final PathFilter LZOP_DEFAULT_SUFFIX_FILTER = new PathFilter() { + @Override + public boolean accept(Path path) + { + return path.toString().endsWith(".lzo"); + } + }; + + private static final PathFilter LZOP_INDEX_DEFAULT_SUFFIX_FILTER = new PathFilter() { + @Override + public boolean accept(Path path) + { + return path.toString().endsWith(".lzo.index"); + } + }; + static { DateTimeParser[] timestampWithoutTimeZoneParser = { DateTimeFormat.forPattern("yyyy-M-d").getParser(), @@ -187,7 +206,7 @@ private HiveUtil() // propagate serialization configuration to getRecordReader schema.stringPropertyNames().stream() - .filter(name -> name.startsWith("serialization.")) + .filter(name -> name.startsWith("serialization.") || name.startsWith("elephantbird.")) .forEach(name -> jobConf.set(name, schema.getProperty(name))); // add Airlift LZO and LZOP to head of codecs list so as to not override existing entries @@ -261,6 +280,11 @@ public static void setReadColumns(Configuration configuration, List rea return MapredParquetInputFormat.class; } + // Remove this after https://github.com/twitter/elephant-bird/pull/481 is included in a release + if ("com.twitter.elephantbird.mapred.input.HiveMultiInputFormat".equals(inputFormatName)) { + return ThriftGeneralInputFormat.class; + } + Class clazz = conf.getClassByName(inputFormatName); return (Class>) clazz.asSubclass(InputFormat.class); } @@ -313,6 +337,21 @@ public static boolean isSplittable(InputFormat inputFormat, FileSystem fil } } + public static boolean isLzopCompressedFile(Path filePath) + { + return LZOP_DEFAULT_SUFFIX_FILTER.accept(filePath); + } + + public static boolean isLzopIndexFile(Path filePath) + { + return LZOP_INDEX_DEFAULT_SUFFIX_FILTER.accept(filePath); + } + + public static Path getLzopIndexPath(Path lzoPath) + { + return lzoPath.suffix(LZO_INDEX_SUFFIX); + } + public static StructObjectInspector getTableObjectInspector(Properties schema) { return getTableObjectInspector(getDeserializer(schema)); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java index 8a161a9d468be..9d4fc92bfe8ee 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveWriteUtils.java @@ -528,6 +528,17 @@ public static boolean isViewFileSystem(HdfsContext context, HdfsEnvironment hdfs } } + public static boolean isHDFSCompatibleViewFileSystem(HdfsContext context, HdfsEnvironment hdfsEnvironment, Path path) + { + try { + return getRawFileSystem(hdfsEnvironment.getFileSystem(context, path)) + .getClass().getName().equals("org.apache.hadoop.fs.viewfs.HDFSCompatibleViewFileSystem"); + } + catch (IOException e) { + throw new PrestoException(HIVE_FILESYSTEM_ERROR, "Failed checking path: " + path, e); + } + } + private static FileSystem getRawFileSystem(FileSystem fileSystem) { if (fileSystem instanceof FilterFileSystem) { @@ -556,6 +567,11 @@ public static Path createTemporaryPath(HdfsContext context, HdfsEnvironment hdfs temporaryPrefix = ".hive-staging"; } + // use relative temporary directory on HDFSCompatibleViewFileSystem + if (isHDFSCompatibleViewFileSystem(context, hdfsEnvironment, targetPath)) { + temporaryPrefix = "../.hive-staging"; + } + // create a temporary directory on the same filesystem Path temporaryRoot = new Path(targetPath, temporaryPrefix); Path temporaryPath = new Path(temporaryRoot, randomUUID().toString()); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/RetryDriver.java b/presto-hive/src/main/java/com/facebook/presto/hive/RetryDriver.java index 911ee8f3712ba..22e1a8127817d 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/RetryDriver.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/RetryDriver.java @@ -137,16 +137,18 @@ public V run(String callableName, Callable callable) return callable.call(); } catch (Exception e) { + log.debug("Failed on executing %s with attempt %d, Exception: %s", callableName, attempt, e.getMessage()); e = exceptionMapper.apply(e); for (Class clazz : exceptionWhiteList) { if (clazz.isInstance(e)) { + log.debug("Exception is in whitelist."); throw e; } } if (attempt >= maxAttempts || Duration.nanosSince(startTime).compareTo(maxRetryTime) >= 0) { + log.debug("Maximum attempts or maximum retry time reached. attempt: %d, maxAttempts: %d, duration: [%s] maxRetryTime: [%s]", attempt, maxAttempts, Duration.nanosSince(startTime).toString(), maxRetryTime.toString()); throw e; } - log.debug("Failed on executing %s with attempt %d, will retry. Exception: %s", callableName, attempt, e.getMessage()); int delayInMs = (int) Math.min(minSleepTime.toMillis() * Math.pow(scaleFactor, attempt - 1), maxSleepTime.toMillis()); int jitter = ThreadLocalRandom.current().nextInt(Math.max(1, (int) (delayInMs * 0.1))); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreModule.java b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreModule.java index 62d94b3875ba0..1ba8ea19281de 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreModule.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/metastore/thrift/ThriftMetastoreModule.java @@ -16,17 +16,21 @@ import com.facebook.presto.hive.ForCachingHiveMetastore; import com.facebook.presto.hive.metastore.CachingHiveMetastore; import com.facebook.presto.hive.metastore.ExtendedHiveMetastore; +import com.facebook.presto.twitter.hive.MetastoreStaticClusterModule; +import com.facebook.presto.twitter.hive.MetastoreZkDiscoveryBasedModule; +import com.facebook.presto.twitter.hive.PooledHiveMetastoreClientFactory; +import com.facebook.presto.twitter.hive.ZookeeperServersetMetastoreConfig; import com.google.inject.Binder; -import com.google.inject.Module; import com.google.inject.Scopes; +import io.airlift.configuration.AbstractConfigurationAwareModule; -import static io.airlift.configuration.ConfigBinder.configBinder; +import static io.airlift.configuration.ConditionalModule.installModuleIf; import static java.util.Objects.requireNonNull; import static org.weakref.jmx.ObjectNames.generatedNameOf; import static org.weakref.jmx.guice.ExportBinder.newExporter; public class ThriftMetastoreModule - implements Module + extends AbstractConfigurationAwareModule { private final String connectorId; @@ -36,11 +40,11 @@ public ThriftMetastoreModule(String connectorId) } @Override - public void configure(Binder binder) + public void setup(Binder binder) { binder.bind(HiveMetastoreClientFactory.class).in(Scopes.SINGLETON); - binder.bind(HiveCluster.class).to(StaticHiveCluster.class).in(Scopes.SINGLETON); - configBinder(binder).bindConfig(StaticMetastoreConfig.class); + binder.bind(PooledHiveMetastoreClientFactory.class).in(Scopes.SINGLETON); + bindMetastoreClusterModule(); binder.bind(HiveMetastore.class).to(ThriftHiveMetastore.class).in(Scopes.SINGLETON); binder.bind(ExtendedHiveMetastore.class).annotatedWith(ForCachingHiveMetastore.class).to(BridgingHiveMetastore.class).in(Scopes.SINGLETON); @@ -50,4 +54,16 @@ public void configure(Binder binder) newExporter(binder).export(ExtendedHiveMetastore.class) .as(generatedNameOf(CachingHiveMetastore.class, connectorId)); } + + private void bindMetastoreClusterModule() + { + install(installModuleIf( + ZookeeperServersetMetastoreConfig.class, + zkMetastoreConfig -> zkMetastoreConfig.getZookeeperServerHostAndPort() == null, + new MetastoreStaticClusterModule())); + install(installModuleIf( + ZookeeperServersetMetastoreConfig.class, + zkMetastoreConfig -> zkMetastoreConfig.getZookeeperServerHostAndPort() != null, + new MetastoreZkDiscoveryBasedModule())); + } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcPageSourceFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcPageSourceFactory.java index c40ebcf00f44a..215ee05a69dcb 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcPageSourceFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcPageSourceFactory.java @@ -263,7 +263,7 @@ private static List getPhysicalHiveColumnHandles(List field = fields.get(fieldId); int fieldIndex; if (useParquetColumnNames) { - fieldIndex = getFieldIndex(fileSchema, columnNames.get(fieldId)); + fieldIndex = findFieldIndexByName(fileSchema, columnNames.get(fieldId)); } else { fieldIndex = hiveColumnIndexes[fieldId]; diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetPageSourceFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetPageSourceFactory.java index 5dd5a33f081be..bda7893c45483 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetPageSourceFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetPageSourceFactory.java @@ -54,6 +54,7 @@ import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR; import static com.facebook.presto.hive.HiveErrorCode.HIVE_CANNOT_OPEN_SPLIT; import static com.facebook.presto.hive.HiveErrorCode.HIVE_MISSING_DATA; +import static com.facebook.presto.hive.HiveSessionProperties.isParquetNestedFieldsProjectionPushdownEnabled; import static com.facebook.presto.hive.HiveSessionProperties.isParquetOptimizedReaderEnabled; import static com.facebook.presto.hive.HiveSessionProperties.isParquetPredicatePushdownEnabled; import static com.facebook.presto.hive.HiveSessionProperties.isUseParquetColumnNames; @@ -61,7 +62,7 @@ import static com.facebook.presto.hive.parquet.HdfsParquetDataSource.buildHdfsParquetDataSource; import static com.facebook.presto.hive.parquet.ParquetTypeUtils.getColumnIO; import static com.facebook.presto.hive.parquet.ParquetTypeUtils.getDescriptors; -import static com.facebook.presto.hive.parquet.ParquetTypeUtils.getParquetType; +import static com.facebook.presto.hive.parquet.ParquetTypeUtils.getPrunedParquetType; import static com.facebook.presto.hive.parquet.predicate.ParquetPredicateUtils.buildParquetPredicate; import static com.facebook.presto.hive.parquet.predicate.ParquetPredicateUtils.getParquetTupleDomain; import static com.facebook.presto.hive.parquet.predicate.ParquetPredicateUtils.predicateMatches; @@ -123,6 +124,7 @@ public Optional createPageSource( schema, columns, isUseParquetColumnNames(session), + isParquetNestedFieldsProjectionPushdownEnabled(session), typeManager, isParquetPredicatePushdownEnabled(session), effectivePredicate, @@ -140,6 +142,7 @@ public static ParquetPageSource createParquetPageSource( Properties schema, List columns, boolean useParquetColumnNames, + boolean pruneNestedFields, TypeManager typeManager, boolean predicatePushdownEnabled, TupleDomain effectivePredicate, @@ -158,7 +161,7 @@ public static ParquetPageSource createParquetPageSource( List fields = columns.stream() .filter(column -> column.getColumnType() == REGULAR) - .map(column -> getParquetType(column, fileSchema, useParquetColumnNames)) + .map(column -> getPrunedParquetType(column, fileSchema, useParquetColumnNames, pruneNestedFields)) .filter(Objects::nonNull) .collect(toList()); diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java index 88c8bb895dc27..280005788e945 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetTypeUtils.java @@ -26,6 +26,8 @@ import com.facebook.presto.spi.type.TimestampType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.VarcharType; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import parquet.column.ColumnDescriptor; import parquet.column.Encoding; import parquet.io.ColumnIO; @@ -36,17 +38,21 @@ import parquet.io.ParquetDecodingException; import parquet.io.PrimitiveColumnIO; import parquet.schema.DecimalMetadata; +import parquet.schema.GroupType; import parquet.schema.MessageType; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; +import java.util.Set; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Optional.empty; import static parquet.schema.OriginalType.DECIMAL; import static parquet.schema.Type.Repetition.REPEATED; @@ -214,10 +220,91 @@ public static int getFieldIndex(MessageType fileSchema, String name) } } + public static parquet.schema.Type getPrunedParquetType(HiveColumnHandle column, MessageType messageType, boolean useParquetColumnNames, boolean pruneNestedFields) + { + parquet.schema.Type originalType = getParquetType(column, messageType, useParquetColumnNames); + if (pruneNestedFields && column.getFieldSet().isPresent()) { + return pruneParquetType(originalType, column.getFieldSet().get()); + } + + return originalType; + } + + private static parquet.schema.Type pruneParquetType(parquet.schema.Type type, Set requiredFields) + { + if (requiredFields.isEmpty()) { + return type; + } + + if (type.isPrimitive()) { + return type; + } + + Map> fields = groupFields(requiredFields); + + List newFields = fields.entrySet().stream() + .map(entry -> pruneParquetType(findParquetTypeByName(type.asGroupType(), entry.getKey()), entry.getValue())) + .collect(toImmutableList()); + + return type.asGroupType().withNewFields(newFields); + } + + private static parquet.schema.Type findParquetTypeByName(GroupType groupType, String name) + { + parquet.schema.Type type = getParquetTypeByName(groupType, name); + + // when a parquet field is a hive keyword we append an _ to it in hive. When doing + // a name-based lookup, we need to strip it off again if we didn't get a direct match. + if (type == null && name.endsWith("_")) { + type = getParquetTypeByName(groupType, name.substring(0, name.length() - 1)); + } + + return type; + } + + private static parquet.schema.Type getParquetTypeByName(GroupType groupType, String fieldName) + { + if (groupType.containsField(fieldName)) { + return groupType.getType(fieldName); + } + + for (parquet.schema.Type type : groupType.getFields()) { + if (type.getName().equalsIgnoreCase(fieldName)) { + return type; + } + } + + return null; + } + + private static Map> groupFields(Set requiredFields) + { + Map> fields = new HashMap<>(); + for (String field : requiredFields) { + String[] path = field.split("\\.", 2); + String fieldName = path[0]; + if (fields.containsKey(fieldName) && fields.get(fieldName).isEmpty()) { + continue; + } + Set nestedField = path.length == 1 ? ImmutableSet.of() : ImmutableSet.of(path[1]); + if (fields.containsKey(fieldName) && nestedField.isEmpty()) { + fields.get(fieldName).clear(); + } + else if (fields.containsKey(fieldName)) { + fields.get(fieldName).addAll(nestedField); + } + else { + fields.put(fieldName, new HashSet<>(nestedField)); + } + } + + return ImmutableMap.copyOf(fields); + } + public static parquet.schema.Type getParquetType(HiveColumnHandle column, MessageType messageType, boolean useParquetColumnNames) { if (useParquetColumnNames) { - return getParquetTypeByName(column.getName(), messageType); + return findParquetTypeByName(column, messageType); } if (column.getHiveColumnIndex() < messageType.getFieldCount()) { @@ -226,6 +313,56 @@ public static parquet.schema.Type getParquetType(HiveColumnHandle column, Messag return null; } + /** + * Find the column type by name using returning the first match with the following logic: + *
    + *
  • direct match
    • + *
  • case-insensitive match
    • + *
  • if the name ends with _, remove it and direct match
    • + *
  • if the name ends with _, remove it and case-insensitive match
    • + *
+ */ + private static parquet.schema.Type findParquetTypeByName(HiveColumnHandle column, MessageType messageType) + { + String name = column.getName(); + parquet.schema.Type type = getParquetTypeByName(name, messageType); + + // when a parquet field is a hive keyword we append an _ to it in hive. When doing + // a name-based lookup, we need to strip it off again if we didn't get a direct match. + if (type == null && name.endsWith("_")) { + type = getParquetTypeByName(name.substring(0, name.length() - 1), messageType); + } + return type; + } + + // Find the column index by name following the same logic as findParquetTypeByName + public static int findFieldIndexByName(MessageType fileSchema, String name) + { + // direct match and case-insensitive match + int fieldIndex = getFieldIndex(fileSchema, name); + + // when a parquet field is a hive keyword we append an _ to it in hive. + // try remove _ and direct match / case-insensitive match again + if (fieldIndex == -1 && name.endsWith("_")) { + fieldIndex = getFieldIndex(fileSchema, name.substring(0, name.length() - 1)); + } + + return fieldIndex; + } + + // Find the ColumnIO by name following the same logic as findParquetTypeByName + public static ColumnIO findColumnIObyName(GroupColumnIO groupColumnIO, String name) + { + // direct match and case-insensitive match + ColumnIO columnIO = lookupColumnByName(groupColumnIO, name); + + if (columnIO == null && name.endsWith("_")) { + columnIO = lookupColumnByName(groupColumnIO, name.substring(0, name.length() - 1)); + } + + return columnIO; + } + public static ParquetEncoding getParquetEncoding(Encoding encoding) { switch (encoding) { diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/HiveFileIterator.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/HiveFileIterator.java index bea9f9ddabbcc..99b6845a3a6ed 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/HiveFileIterator.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/HiveFileIterator.java @@ -97,7 +97,14 @@ protected LocatedFileStatus computeNext() if (paths.isEmpty()) { return endOfData(); } - remoteIterator = getLocatedFileStatusRemoteIterator(paths.removeFirst()); + try { + remoteIterator = getLocatedFileStatusRemoteIterator(paths.removeFirst()); + } + catch (PrestoException e) { + if (!e.getErrorCode().equals(HIVE_FILE_NOT_FOUND.toErrorCode())) { + throw e; + } + } } } diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/util/InternalHiveSplitFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/util/InternalHiveSplitFactory.java index dd19146b5800f..9448b8a27e5bc 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/util/InternalHiveSplitFactory.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/util/InternalHiveSplitFactory.java @@ -41,6 +41,7 @@ import java.util.Properties; import static com.facebook.presto.hive.HiveColumnHandle.isPathColumnHandle; +import static com.facebook.presto.hive.HiveUtil.isLzopIndexFile; import static com.facebook.presto.hive.HiveUtil.isSplittable; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -239,6 +240,10 @@ private static Optional getPathDomain(TupleDomain effe private static boolean pathMatchesPredicate(Optional pathDomain, String path) { + if (isLzopIndexFile(new Path(path))) { + return false; + } + if (!pathDomain.isPresent()) { return true; } diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/MetastoreStaticClusterModule.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/MetastoreStaticClusterModule.java new file mode 100644 index 0000000000000..8751b5bc474f6 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/MetastoreStaticClusterModule.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive; + +import com.facebook.presto.hive.metastore.thrift.HiveCluster; +import com.facebook.presto.hive.metastore.thrift.StaticHiveCluster; +import com.facebook.presto.hive.metastore.thrift.StaticMetastoreConfig; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Scopes; + +import static io.airlift.configuration.ConfigBinder.configBinder; + +public class MetastoreStaticClusterModule + implements Module +{ + @Override + public void configure(Binder binder) + { + binder.bind(HiveCluster.class).to(StaticHiveCluster.class).in(Scopes.SINGLETON); + configBinder(binder).bindConfig(StaticMetastoreConfig.class); + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/MetastoreZkDiscoveryBasedModule.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/MetastoreZkDiscoveryBasedModule.java new file mode 100644 index 0000000000000..5459eca553054 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/MetastoreZkDiscoveryBasedModule.java @@ -0,0 +1,32 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive; + +import com.facebook.presto.hive.metastore.thrift.HiveCluster; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Scopes; + +import static io.airlift.configuration.ConfigBinder.configBinder; + +public class MetastoreZkDiscoveryBasedModule + implements Module +{ + @Override + public void configure(Binder binder) + { + binder.bind(HiveCluster.class).to(ZookeeperServersetHiveCluster.class).in(Scopes.SINGLETON); + configBinder(binder).bindConfig(ZookeeperServersetMetastoreConfig.class); + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java new file mode 100644 index 0000000000000..270e313e75114 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/PooledHiveMetastoreClientFactory.java @@ -0,0 +1,92 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive; + +import com.facebook.presto.hive.HiveClientConfig; +import com.facebook.presto.hive.authentication.HiveMetastoreAuthentication; +import com.facebook.presto.hive.metastore.thrift.HiveMetastoreClient; +import com.facebook.presto.hive.metastore.thrift.ThriftHiveMetastoreClient; +import com.facebook.presto.twitter.hive.util.PooledTTransportFactory; +import com.facebook.presto.twitter.hive.util.TTransportPool; +import com.google.common.net.HostAndPort; +import io.airlift.units.Duration; +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; + +import javax.annotation.Nullable; +import javax.inject.Inject; + +import static java.lang.Math.toIntExact; +import static java.util.Objects.requireNonNull; + +public class PooledHiveMetastoreClientFactory +{ + private final HostAndPort socksProxy; + private final int timeoutMillis; + private final HiveMetastoreAuthentication metastoreAuthentication; + private final TTransportPool transportPool; + + public PooledHiveMetastoreClientFactory(@Nullable HostAndPort socksProxy, + Duration timeout, + HiveMetastoreAuthentication metastoreAuthentication, + int maxTransport, + long idleTimeout, + long transportEvictInterval, + int evictNumTests) + { + this.socksProxy = socksProxy; + this.timeoutMillis = toIntExact(timeout.toMillis()); + this.metastoreAuthentication = requireNonNull(metastoreAuthentication, "metastoreAuthentication is null"); + GenericObjectPoolConfig poolConfig = new GenericObjectPoolConfig(); + poolConfig.setMaxIdle(maxTransport); + poolConfig.setMaxTotal(maxTransport); + poolConfig.setMinEvictableIdleTimeMillis(idleTimeout); + poolConfig.setTimeBetweenEvictionRunsMillis(transportEvictInterval); + poolConfig.setNumTestsPerEvictionRun(evictNumTests); + this.transportPool = new TTransportPool(poolConfig); + } + + @Inject + public PooledHiveMetastoreClientFactory(HiveClientConfig config, + ZookeeperServersetMetastoreConfig zkConfig, + HiveMetastoreAuthentication metastoreAuthentication) + { + this(config.getMetastoreSocksProxy(), + config.getMetastoreTimeout(), + metastoreAuthentication, + zkConfig.getMaxTransport(), + zkConfig.getTransportIdleTimeout(), + zkConfig.getTransportEvictInterval(), + zkConfig.getTransportEvictNumTests()); + } + + public HiveMetastoreClient create(String host, int port) + throws TTransportException + { + try { + TTransport transport = transportPool.borrowObject(host, port); + if (transport == null) { + transport = transportPool.borrowObject(host, port, + new PooledTTransportFactory(transportPool, + host, port, socksProxy, + timeoutMillis, metastoreAuthentication)); + } + return new ThriftHiveMetastoreClient(transport); + } + catch (Exception e) { + throw new TTransportException(String.format("%s: %s", host, e.getMessage()), e.getCause()); + } + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperMetastoreMonitor.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperMetastoreMonitor.java new file mode 100644 index 0000000000000..cf8df021c1a1b --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperMetastoreMonitor.java @@ -0,0 +1,119 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive; + +import com.google.common.net.HostAndPort; +import io.airlift.log.Logger; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.framework.recipes.cache.PathChildrenCache; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.curator.utils.ZKPaths; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; + +public class ZookeeperMetastoreMonitor + implements PathChildrenCacheListener +{ + public static final Logger log = Logger.get(ZookeeperMetastoreMonitor.class); + private CuratorFramework client; + private PathChildrenCache cache; + private ConcurrentMap servers; // (Node_Name->HostAndPort) + + public ZookeeperMetastoreMonitor(String zkServer, String watchPath, int maxRetries, int retrySleepTime) + throws Exception + { + client = CuratorFrameworkFactory.newClient(zkServer, new ExponentialBackoffRetry(retrySleepTime, maxRetries)); + client.start(); + + cache = new PathChildrenCache(client, watchPath, true); // true indicating cache node contents in addition to the stat + try { + cache.start(); + } + catch (Exception ex) { + throw new RuntimeException("Curator PathCache Creation failed: " + ex.getMessage()); + } + + cache.getListenable().addListener(this); + servers = new ConcurrentHashMap<>(); + } + + public void close() + { + client.close(); + + try { + cache.close(); + } + catch (IOException ex) { + // do nothing + } + } + + public List getServers() + { + return servers.values().stream().collect(Collectors.toList()); + } + + private HostAndPort deserialize(byte[] bytes) + { + String serviceEndpoint = "serviceEndpoint"; + JSONObject data = (JSONObject) JSONValue.parse(new String(bytes)); + if (data != null && data.containsKey(serviceEndpoint)) { + Map hostPortMap = (Map) data.get(serviceEndpoint); + String host = hostPortMap.get("host").toString(); + int port = Integer.parseInt(hostPortMap.get("port").toString()); + return HostAndPort.fromParts(host, port); + } + else { + log.warn("failed to deserialize child node data"); + throw new IllegalArgumentException("No host:port found"); + } + } + + @Override + public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception + { + switch (event.getType()) { + case CHILD_ADDED: + case CHILD_UPDATED: { + HostAndPort hostPort = deserialize(event.getData().getData()); + String node = ZKPaths.getNodeFromPath(event.getData().getPath()); + log.info("child updated: " + node + ": " + hostPort); + servers.put(node, hostPort); + break; + } + + case CHILD_REMOVED: { + String node = ZKPaths.getNodeFromPath(event.getData().getPath()); + log.info("child removed: " + node); + servers.remove(node); + break; + } + + default: + log.info("connection state changed: " + event.getType()); + break; + } + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java new file mode 100644 index 0000000000000..c91912f0ff916 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetHiveCluster.java @@ -0,0 +1,67 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive; + +import com.facebook.presto.hive.metastore.thrift.HiveCluster; +import com.facebook.presto.hive.metastore.thrift.HiveMetastoreClient; +import com.google.common.net.HostAndPort; +import io.airlift.log.Logger; +import org.apache.thrift.transport.TTransportException; + +import javax.inject.Inject; + +import java.util.Collections; +import java.util.List; + +import static java.util.Objects.requireNonNull; + +public class ZookeeperServersetHiveCluster + implements HiveCluster +{ + private static final Logger log = Logger.get(ZookeeperServersetHiveCluster.class); + private final PooledHiveMetastoreClientFactory clientFactory; + private ZookeeperMetastoreMonitor zkMetastoreMonitor; + + @Inject + public ZookeeperServersetHiveCluster(ZookeeperServersetMetastoreConfig config, PooledHiveMetastoreClientFactory clientFactory) + throws Exception + { + String zkServerHostAndPort = requireNonNull(config.getZookeeperServerHostAndPort(), "zkServerHostAndPort is null"); + String zkMetastorePath = requireNonNull(config.getZookeeperMetastorePath(), "zkMetastorePath is null"); + int zkRetries = requireNonNull(config.getZookeeperMaxRetries(), "zkMaxRetried is null"); + int zkRetrySleepTime = requireNonNull(config.getZookeeperRetrySleepTime(), "zkRetrySleepTime is null"); + this.clientFactory = requireNonNull(clientFactory, "clientFactory is null"); + this.zkMetastoreMonitor = new ZookeeperMetastoreMonitor(zkServerHostAndPort, zkMetastorePath, zkRetries, zkRetrySleepTime); + } + + @Override + public HiveMetastoreClient createMetastoreClient() + { + List metastores = zkMetastoreMonitor.getServers(); + Collections.shuffle(metastores); + TTransportException lastException = null; + for (HostAndPort metastore : metastores) { + try { + log.info("Connecting to metastore at: %s", metastore.toString()); + return clientFactory.create(metastore.getHost(), metastore.getPort()); + } + catch (TTransportException e) { + log.debug("Failed connecting to Hive metastore at: %s", metastore.toString()); + lastException = e; + } + } + + throw new RuntimeException("Failed connecting to Hive metastore.", lastException); + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetMetastoreConfig.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetMetastoreConfig.java new file mode 100644 index 0000000000000..65b424b6c437e --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/ZookeeperServersetMetastoreConfig.java @@ -0,0 +1,136 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive; + +import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigDescription; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +public class ZookeeperServersetMetastoreConfig +{ + private String zookeeperServerHostAndPort; + private String zookeeperMetastorePath; + private int zookeeperRetrySleepTime = 500; // ms + private int zookeeperMaxRetries = 3; + private int maxTransport = 128; + private long transportIdleTimeout = 300_000L; + private long transportEvictInterval = 10_000L; + private int transportEvictNumTests = 3; + + public String getZookeeperServerHostAndPort() + { + return zookeeperServerHostAndPort; + } + + @Config("hive.metastore.zookeeper.uri") + @ConfigDescription("Zookeeper Host and Port") + public ZookeeperServersetMetastoreConfig setZookeeperServerHostAndPort(String zookeeperServerHostAndPort) + { + this.zookeeperServerHostAndPort = zookeeperServerHostAndPort; + return this; + } + + public String getZookeeperMetastorePath() + { + return zookeeperMetastorePath; + } + + @Config("hive.metastore.zookeeper.path") + @ConfigDescription("Hive metastore Zookeeper path") + public ZookeeperServersetMetastoreConfig setZookeeperMetastorePath(String zkPath) + { + this.zookeeperMetastorePath = zkPath; + return this; + } + + @NotNull + public int getZookeeperRetrySleepTime() + { + return zookeeperRetrySleepTime; + } + + @Config("hive.metastore.zookeeper.retry.sleeptime") + @ConfigDescription("Zookeeper sleep time between reties") + public ZookeeperServersetMetastoreConfig setZookeeperRetrySleepTime(int zookeeperRetrySleepTime) + { + this.zookeeperRetrySleepTime = zookeeperRetrySleepTime; + return this; + } + + @Min(1) + public int getZookeeperMaxRetries() + { + return zookeeperMaxRetries; + } + + @Config("hive.metastore.zookeeper.max.retries") + @ConfigDescription("Zookeeper max reties") + public ZookeeperServersetMetastoreConfig setZookeeperMaxRetries(int zookeeperMaxRetries) + { + this.zookeeperMaxRetries = zookeeperMaxRetries; + return this; + } + + @Min(1) + public int getMaxTransport() + { + return maxTransport; + } + + @Config("hive.metastore.max-transport-num") + public ZookeeperServersetMetastoreConfig setMaxTransport(int maxTransport) + { + this.maxTransport = maxTransport; + return this; + } + + public long getTransportIdleTimeout() + { + return transportIdleTimeout; + } + + @Config("hive.metastore.transport-idle-timeout") + public ZookeeperServersetMetastoreConfig setTransportIdleTimeout(long transportIdleTimeout) + { + this.transportIdleTimeout = transportIdleTimeout; + return this; + } + + public long getTransportEvictInterval() + { + return transportEvictInterval; + } + + @Config("hive.metastore.transport-eviction-interval") + public ZookeeperServersetMetastoreConfig setTransportEvictInterval(long transportEvictInterval) + { + this.transportEvictInterval = transportEvictInterval; + return this; + } + + @Min(0) + public int getTransportEvictNumTests() + { + return transportEvictNumTests; + } + + @Config("hive.metastore.transport-eviction-num-tests") + public ZookeeperServersetMetastoreConfig setTransportEvictNumTests(int transportEvictNumTests) + { + this.transportEvictNumTests = transportEvictNumTests; + return this; + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/HiveThriftFieldIdGroup.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/HiveThriftFieldIdGroup.java new file mode 100644 index 0000000000000..6b43f7244889f --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/HiveThriftFieldIdGroup.java @@ -0,0 +1,94 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive.thrift; + +import com.facebook.presto.spi.type.Type; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static com.facebook.presto.hive.HiveUtil.isStructuralType; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.Maps.immutableEntry; +import static java.util.Objects.requireNonNull; + +public class HiveThriftFieldIdGroup +{ + private final Map fields; + + public HiveThriftFieldIdGroup(Map fields) + { + this.fields = requireNonNull(fields, "fields is null"); + } + + public HiveThriftFieldIdGroup getFieldIdGroup(short thriftId) + { + return fields.get(thriftId); + } + + public Set getFieldIds() + { + return ImmutableSet.copyOf(fields.keySet()); + } + + public static HiveThriftFieldIdGroup create(Type type, Set requiredFields, ThriftFieldIdResolver fieldIdResolver) + { + if (requiredFields.isEmpty() || (!isStructuralType(type))) { + return new HiveThriftFieldIdGroup(ImmutableMap.of()); + } + + Map> fields = groupFields(requiredFields); + + List fieldNames = type.getTypeParameters().stream() + .map(Type::getDisplayName) + .collect(toImmutableList()); + + return new HiveThriftFieldIdGroup(fields.entrySet().stream() + .filter(entry -> fieldNames.contains(entry.getKey())) + .map(entry -> immutableEntry(fieldNames.indexOf(entry.getKey()), entry.getValue())) + .collect(toImmutableMap(entry -> fieldIdResolver.getThriftId(entry.getKey()), + entry -> create(type.getTypeParameters().get(entry.getKey()), + entry.getValue(), fieldIdResolver.getNestedResolver(entry.getKey()))))); + } + + private static Map> groupFields(Set requiredFields) + { + Map> fields = new HashMap<>(); + for (String field : requiredFields) { + String[] path = field.split("\\.", 2); + String fieldName = path[0]; + if (fields.containsKey(fieldName) && fields.get(fieldName).isEmpty()) { + continue; + } + Set nestedField = path.length == 1 ? ImmutableSet.of() : ImmutableSet.of(path[1]); + if (fields.containsKey(fieldName) && nestedField.isEmpty()) { + fields.get(fieldName).clear(); + } + else if (fields.containsKey(fieldName)) { + fields.get(fieldName).addAll(nestedField); + } + else { + fields.put(fieldName, new HashSet<>(nestedField)); + } + } + + return ImmutableMap.copyOf(fields); + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/HiveThriftFieldIdResolver.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/HiveThriftFieldIdResolver.java new file mode 100644 index 0000000000000..077a409fd2a2e --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/HiveThriftFieldIdResolver.java @@ -0,0 +1,130 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive.thrift; + +import com.fasterxml.jackson.databind.JsonNode; + +import java.util.HashMap; +import java.util.Map; + +import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; +import static com.facebook.presto.hive.HiveUtil.checkCondition; +import static com.google.common.base.MoreObjects.toStringHelper; + +/** + * Resolve the translation of continuous hive ids to discontinuous thrift ids by using a json property. + * Example: + * We have the thrift definition: + * + * struct Name { + * 1: string first, + * 2: string last + * } + * struct Person { + * 1: Name name, + * 3: String phone + * } + * + * Hive table for Person: + * + * +---------+-------------+----------------------------------+-----------------+ + * | hive id | column name | type | thrift field id | + * +---------+-------------+----------------------------------+-----------------+ + * | 0 | name | struct | 1 | + * +---------+-------------+----------------------------------+-----------------+ + * | 1 | phone | string | 3 | + * +---------+-------------+----------------------------------+-----------------+ + * + * The corresponding id mapping object is: + * + * x = { + * '0': { + * '0': 1, + * '1': 2, + * }, + * '1': 3 + * } + * + * The json property is: + * + * {"0":{"0":1,"1":2},"1":3} + */ +public class HiveThriftFieldIdResolver + implements ThriftFieldIdResolver +{ + private final JsonNode root; + private final Map nestedResolvers = new HashMap<>(); + private final Map thriftIds = new HashMap<>(); + + public HiveThriftFieldIdResolver(JsonNode root) + { + this.root = root; + } + + @Override + public short getThriftId(int hiveIndex) + { + if (root == null) { + return (short) (hiveIndex + 1); + } + + Short thriftId = thriftIds.get(hiveIndex); + if (thriftId != null) { + return thriftId; + } + else { + JsonNode child = root.get(String.valueOf(hiveIndex)); + checkCondition(child != null, HIVE_INVALID_METADATA, "Missed json value for hiveIndex: %s, root: %s", hiveIndex, root); + if (child.isNumber()) { + thriftId = (short) child.asInt(); + } + else { + checkCondition(child.get("id") != null, HIVE_INVALID_METADATA, "Missed id for hiveIndex: %s, root: %s", hiveIndex, root); + thriftId = (short) child.get("id").asInt(); + } + thriftIds.put(hiveIndex, thriftId); + return thriftId; + } + } + + @Override + public ThriftFieldIdResolver getNestedResolver(int hiveIndex) + { + if (root == null) { + return this; + } + + ThriftFieldIdResolver nestedResolver = nestedResolvers.get(hiveIndex); + if (nestedResolver != null) { + return nestedResolver; + } + else { + JsonNode child = root.get(String.valueOf(hiveIndex)); + checkCondition(child != null, HIVE_INVALID_METADATA, "Missed json value for hiveIndex: %s, root: %s", hiveIndex, root); + nestedResolver = new HiveThriftFieldIdResolver(child); + nestedResolvers.put(hiveIndex, nestedResolver); + return nestedResolver; + } + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("root", root) + .add("nestedResolvers", nestedResolvers) + .add("thriftIds", thriftIds) + .toString(); + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/HiveThriftFieldIdResolverFactory.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/HiveThriftFieldIdResolverFactory.java new file mode 100644 index 0000000000000..c1c736a9f4536 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/HiveThriftFieldIdResolverFactory.java @@ -0,0 +1,49 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive.thrift; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.airlift.log.Logger; + +import java.io.IOException; +import java.util.Properties; + +public class HiveThriftFieldIdResolverFactory + implements ThriftFieldIdResolverFactory +{ + private static final Logger log = Logger.get(HiveThriftFieldIdResolverFactory.class); + private static final ObjectMapper objectMapper = new ObjectMapper(); + public static final String THRIFT_FIELD_ID_JSON = "thrift.field.id.json"; + // The default resolver which returns thrift id as hive id plus one + public static final ThriftFieldIdResolver HIVE_THRIFT_FIELD_ID_DEFAULT_RESOLVER = new HiveThriftFieldIdResolver(null); + + public ThriftFieldIdResolver createResolver(Properties schema) + { + String jsonData = schema.getProperty(THRIFT_FIELD_ID_JSON); + if (jsonData == null) { + return HIVE_THRIFT_FIELD_ID_DEFAULT_RESOLVER; + } + + try { + JsonNode root = objectMapper.readTree(jsonData); + return new HiveThriftFieldIdResolver(root); + } + catch (IOException e) { + log.debug(e, "Failed to create an optimized thrift id resolver, json string: %s, schema: %s. Will use a default resolver.", jsonData, schema); + } + + return HIVE_THRIFT_FIELD_ID_DEFAULT_RESOLVER; + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftFieldIdResolver.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftFieldIdResolver.java new file mode 100644 index 0000000000000..83047e1d953fb --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftFieldIdResolver.java @@ -0,0 +1,20 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive.thrift; + +public interface ThriftFieldIdResolver +{ + ThriftFieldIdResolver getNestedResolver(int hiveIndex); + short getThriftId(int hiveIndex); +} diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftFieldIdResolverFactory.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftFieldIdResolverFactory.java new file mode 100644 index 0000000000000..034308aaa569a --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftFieldIdResolverFactory.java @@ -0,0 +1,21 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive.thrift; + +import java.util.Properties; + +public interface ThriftFieldIdResolverFactory +{ + ThriftFieldIdResolver createResolver(Properties schema); +} diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftGeneralDeserializer.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftGeneralDeserializer.java new file mode 100644 index 0000000000000..9f1a8d7d77606 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftGeneralDeserializer.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive.thrift; + +import com.twitter.elephantbird.mapreduce.io.ThriftWritable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Writable; +import org.apache.thrift.TException; + +import java.util.Properties; + +import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; +import static com.facebook.presto.hive.HiveErrorCode.HIVE_UNKNOWN_ERROR; +import static com.facebook.presto.hive.HiveUtil.checkCondition; +import static org.apache.hadoop.hive.serde.Constants.SERIALIZATION_CLASS; + +public class ThriftGeneralDeserializer +{ + private static final String REQUIRED_SERIALIZATION_CLASS = ThriftGenericRow.class.getName(); + public ThriftGeneralDeserializer(Configuration conf, Properties properties) + { + String thriftClassName = properties.getProperty(SERIALIZATION_CLASS, null); + checkCondition(thriftClassName != null, HIVE_INVALID_METADATA, "Table or partition is missing Hive deserializer property: %s", SERIALIZATION_CLASS); + checkCondition(thriftClassName.equals(REQUIRED_SERIALIZATION_CLASS), HIVE_INVALID_METADATA, SERIALIZATION_CLASS + thriftClassName + " cannot match " + REQUIRED_SERIALIZATION_CLASS); + } + + public ThriftGenericRow deserialize(Writable writable, HiveThriftFieldIdGroup thriftFieldIdGroup) + { + checkCondition(writable instanceof ThriftWritable, HIVE_UNKNOWN_ERROR, "Not an instance of ThriftWritable: " + writable); + ThriftGenericRow row = (ThriftGenericRow) ((ThriftWritable) writable).get(); + try { + row.parse(thriftFieldIdGroup); + } + catch (TException e) { + throw new IllegalStateException("ThriftGenericRow failed to parse values", e); + } + return row; + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftGeneralInputFormat.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftGeneralInputFormat.java new file mode 100644 index 0000000000000..1f9690036dbf9 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftGeneralInputFormat.java @@ -0,0 +1,95 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive.thrift; + +import com.facebook.presto.spi.PrestoException; +import com.twitter.elephantbird.mapred.input.DeprecatedFileInputFormatWrapper; +import com.twitter.elephantbird.mapreduce.input.MultiInputFormat; +import com.twitter.elephantbird.mapreduce.io.BinaryWritable; +import com.twitter.elephantbird.util.TypeRef; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.mapred.FileSplit; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.hadoop.mapred.Reporter; + +import java.io.IOException; + +import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; +import static com.facebook.presto.hive.HiveUtil.checkCondition; +import static com.facebook.presto.hive.HiveUtil.getLzopIndexPath; +import static com.facebook.presto.hive.HiveUtil.isLzopCompressedFile; +import static java.lang.String.format; +import static org.apache.hadoop.hive.serde.Constants.SERIALIZATION_CLASS; + +/** + * Mirror of com.twitter.elephantbird.mapred.input.HiveMultiInputFormat allows to pass the thriftClassName + * directly as a property of JobConfig and check lzo index existence when check splitability. + * PR for twitter/elephant-bird: + * https://github.com/twitter/elephant-bird/pull/481 + * https://github.com/twitter/elephant-bird/pull/485 + * Remove the class once #481 is included in a release + */ +@SuppressWarnings("deprecation") +public class ThriftGeneralInputFormat + extends DeprecatedFileInputFormatWrapper +{ + public ThriftGeneralInputFormat() + { + super(new MultiInputFormat()); + } + + private void initialize(FileSplit split, JobConf job) throws IOException + { + String thriftClassName = job.get(SERIALIZATION_CLASS); + checkCondition(thriftClassName != null, HIVE_INVALID_METADATA, "Table or partition is missing Hive deserializer property: %s", SERIALIZATION_CLASS); + + try { + Class thriftClass = job.getClassByName(thriftClassName); + setInputFormatInstance(new MultiInputFormat(new TypeRef(thriftClass) {})); + } + catch (ClassNotFoundException e) { + throw new PrestoException(HIVE_INVALID_METADATA, format("Failed getting class for %s", thriftClassName)); + } + } + + @Override + public boolean isSplitable(FileSystem fs, Path filename) + { + if (isLzopCompressedFile(filename)) { + Path indexFile = getLzopIndexPath(filename); + try { + return fs.exists(indexFile); + } + catch (IOException e) { + return false; + } + } + return super.isSplitable(fs, filename); + } + + @Override + public RecordReader getRecordReader( + InputSplit split, + JobConf job, + Reporter reporter) + throws IOException + { + initialize((FileSplit) split, job); + return super.getRecordReader(split, job, reporter); + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftGenericRow.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftGenericRow.java new file mode 100644 index 0000000000000..c97f857996f9e --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftGenericRow.java @@ -0,0 +1,231 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive.thrift; + +import io.airlift.log.Logger; +import org.apache.thrift.TBase; +import org.apache.thrift.TException; +import org.apache.thrift.TFieldIdEnum; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TField; +import org.apache.thrift.protocol.TList; +import org.apache.thrift.protocol.TMap; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolUtil; +import org.apache.thrift.protocol.TSet; +import org.apache.thrift.protocol.TType; +import org.apache.thrift.transport.TMemoryInputTransport; +import org.apache.thrift.transport.TTransport; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class ThriftGenericRow + implements TBase +{ + private static final Logger log = Logger.get(ThriftGenericRow.class); + private final Map values = new HashMap<>(); + private byte[] buf; + private int off; + private int len; + + public ThriftGenericRow() + { + } + + public ThriftGenericRow(Map values) + { + this.values.putAll(values); + } + + public class Fields + implements TFieldIdEnum + { + private final short thriftId; + private final String fieldName; + + Fields(short thriftId, String fieldName) + { + this.thriftId = thriftId; + this.fieldName = fieldName; + } + + public short getThriftFieldId() + { + return thriftId; + } + + public String getFieldName() + { + return fieldName; + } + } + + public void read(TProtocol iprot) + throws TException + { + TTransport trans = iprot.getTransport(); + buf = trans.getBuffer(); + off = trans.getBufferPosition(); + TProtocolUtil.skip(iprot, TType.STRUCT); + len = trans.getBufferPosition() - off; + } + + public void parse(HiveThriftFieldIdGroup fieldIdGroup) + throws TException + { + TMemoryInputTransport trans = new TMemoryInputTransport(buf, off, len); + TBinaryProtocol iprot = new TBinaryProtocol(trans); + TField field; + iprot.readStructBegin(); + while (true) { + field = iprot.readFieldBegin(); + if (field.type == TType.STOP) { + break; + } + if (fieldIdGroup != null && fieldIdGroup.getFieldIdGroup(field.id) == null) { + TProtocolUtil.skip(iprot, field.type); + } + else { + values.put(field.id, readElem(iprot, field.type, fieldIdGroup == null ? null : fieldIdGroup.getFieldIdGroup(field.id))); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + } + + private Object readElem(TProtocol iprot, byte type, HiveThriftFieldIdGroup fieldIdGroup) + throws TException + { + switch (type) { + case TType.BOOL: + return iprot.readBool(); + case TType.BYTE: + return iprot.readByte(); + case TType.I16: + return iprot.readI16(); + case TType.ENUM: + case TType.I32: + return iprot.readI32(); + case TType.I64: + return iprot.readI64(); + case TType.DOUBLE: + return iprot.readDouble(); + case TType.STRING: + return iprot.readString(); + case TType.STRUCT: + return readStruct(iprot, fieldIdGroup); + case TType.LIST: + return readList(iprot, fieldIdGroup); + case TType.SET: + return readSet(iprot, fieldIdGroup); + case TType.MAP: + return readMap(iprot, fieldIdGroup); + default: + TProtocolUtil.skip(iprot, type); + return null; + } + } + + private Object readStruct(TProtocol iprot, HiveThriftFieldIdGroup fieldIdGroup) + throws TException + { + ThriftGenericRow elem = new ThriftGenericRow(); + elem.read(iprot); + elem.parse(fieldIdGroup); + return elem; + } + + private Object readList(TProtocol iprot, HiveThriftFieldIdGroup fieldIdGroup) + throws TException + { + TList ilist = iprot.readListBegin(); + List listValue = new ArrayList<>(); + for (int i = 0; i < ilist.size; i++) { + listValue.add(readElem(iprot, ilist.elemType, fieldIdGroup == null ? null : fieldIdGroup.getFieldIdGroup((short) 0))); + } + iprot.readListEnd(); + return listValue; + } + + private Object readSet(TProtocol iprot, HiveThriftFieldIdGroup fieldIdGroup) + throws TException + { + TSet iset = iprot.readSetBegin(); + List setValue = new ArrayList<>(); + for (int i = 0; i < iset.size; i++) { + setValue.add(readElem(iprot, iset.elemType, fieldIdGroup == null ? null : fieldIdGroup.getFieldIdGroup((short) 0))); + } + iprot.readSetEnd(); + return setValue; + } + + private Object readMap(TProtocol iprot, HiveThriftFieldIdGroup fieldIdGroup) + throws TException + { + TMap imap = iprot.readMapBegin(); + Map mapValue = new HashMap<>(); + for (int i = 0; i < imap.size; i++) { + mapValue.put(readElem(iprot, imap.keyType, fieldIdGroup == null ? null : fieldIdGroup.getFieldIdGroup((short) 0)), + readElem(iprot, imap.valueType, fieldIdGroup == null ? null : fieldIdGroup.getFieldIdGroup((short) 1))); + } + iprot.readMapEnd(); + return mapValue; + } + + public Object getFieldValueForThriftId(short thriftId) + { + return values.get(thriftId); + } + + public ThriftGenericRow deepCopy() + { + return new ThriftGenericRow(values); + } + + public void clear() {} + + public Fields fieldForId(int fieldId) + { + return new Fields((short) fieldId, "dummy"); + } + + public Object getFieldValue(Fields field) + { + return values.get(field.thriftId); + } + + public boolean isSet(Fields field) + { + return values.containsKey(field.getThriftFieldId()); + } + + public void setFieldValue(Fields field, Object value) + { + values.put(field.getThriftFieldId(), value); + } + + public void write(TProtocol oprot) + throws TException + { + throw new UnsupportedOperationException("ThriftGenericRow.write is not supported."); + } + + public int compareTo(ThriftGenericRow other) + { + throw new UnsupportedOperationException("ThriftGenericRow.compareTo is not supported."); + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftHiveRecordCursor.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftHiveRecordCursor.java new file mode 100644 index 0000000000000..1382ec7e430f2 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftHiveRecordCursor.java @@ -0,0 +1,731 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive.thrift; + +import com.facebook.presto.hive.HiveColumnHandle; +import com.facebook.presto.hive.HiveType; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.RecordCursor; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.type.DecimalType; +import com.facebook.presto.spi.type.Decimals; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; +import com.google.common.collect.ImmutableMap; +import io.airlift.log.Logger; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.type.HiveChar; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.HiveVarchar; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapred.RecordReader; +import org.joda.time.DateTimeZone; + +import java.io.IOException; +import java.math.BigInteger; +import java.sql.Date; +import java.sql.Timestamp; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +import static com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR; +import static com.facebook.presto.hive.HiveErrorCode.HIVE_CURSOR_ERROR; +import static com.facebook.presto.hive.HiveUtil.closeWithSuppression; +import static com.facebook.presto.hive.HiveUtil.isArrayType; +import static com.facebook.presto.hive.HiveUtil.isMapType; +import static com.facebook.presto.hive.HiveUtil.isRowType; +import static com.facebook.presto.hive.HiveUtil.isStructuralType; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.Chars.isCharType; +import static com.facebook.presto.spi.type.Chars.truncateToLengthAndTrimSpaces; +import static com.facebook.presto.spi.type.DateType.DATE; +import static com.facebook.presto.spi.type.Decimals.rescale; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.RealType.REAL; +import static com.facebook.presto.spi.type.SmallintType.SMALLINT; +import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; +import static com.facebook.presto.spi.type.TinyintType.TINYINT; +import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; +import static com.facebook.presto.spi.type.Varchars.isVarcharType; +import static com.facebook.presto.spi.type.Varchars.truncateToLength; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static java.lang.Float.floatToRawIntBits; +import static java.lang.Math.max; +import static java.lang.Math.min; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +class ThriftHiveRecordCursor + implements RecordCursor +{ + private static final Logger log = Logger.get(ThriftHiveRecordCursor.class); + private static final short NON_EXISTED_THRIFT_ID = (short) -1; + private final RecordReader recordReader; + private final K key; + private final V value; + + private final ThriftGeneralDeserializer deserializer; + + private final Type[] types; + private final HiveType[] hiveTypes; + private final int[] hiveIndexs; + private final short[] thriftIds; + private final HiveThriftFieldIdGroup thriftFieldIdGroup; + + private final boolean[] loaded; + private final boolean[] booleans; + private final long[] longs; + private final double[] doubles; + private final Slice[] slices; + private final Object[] objects; + private final boolean[] nulls; + + private final Path path; + private final long start; + private final long totalBytes; + private final DateTimeZone hiveStorageTimeZone; + + private final ThriftFieldIdResolver thriftFieldIdResolver; + + private long completedBytes; + private ThriftGenericRow rowData; + private boolean closed; + + public ThriftHiveRecordCursor( + RecordReader recordReader, + Path path, + long start, + long totalBytes, + Properties splitSchema, + List columns, + DateTimeZone hiveStorageTimeZone, + TypeManager typeManager, + ThriftFieldIdResolver thriftFieldIdResolver) + { + requireNonNull(recordReader, "recordReader is null"); + requireNonNull(path, "path is null"); + checkArgument(start >= 0, "start is negative"); + checkArgument(totalBytes >= 0, "totalBytes is negative"); + requireNonNull(splitSchema, "splitSchema is null"); + requireNonNull(columns, "columns is null"); + requireNonNull(hiveStorageTimeZone, "hiveStorageTimeZone is null"); + requireNonNull(thriftFieldIdResolver, "thriftFieldIdResolver is null"); + + this.recordReader = recordReader; + this.path = path; + this.start = start; + this.totalBytes = totalBytes; + this.key = recordReader.createKey(); + this.value = recordReader.createValue(); + this.hiveStorageTimeZone = hiveStorageTimeZone; + this.thriftFieldIdResolver = thriftFieldIdResolver; + + this.deserializer = new ThriftGeneralDeserializer(new Configuration(false), splitSchema); + + int size = columns.size(); + + this.types = new Type[size]; + this.hiveTypes = new HiveType[size]; + this.hiveIndexs = new int[size]; + this.thriftIds = new short[size]; + + this.loaded = new boolean[size]; + this.booleans = new boolean[size]; + this.longs = new long[size]; + this.doubles = new double[size]; + this.slices = new Slice[size]; + this.objects = new Object[size]; + this.nulls = new boolean[size]; + + ImmutableMap.Builder thriftFieldIdGroupBuilder = ImmutableMap.builder(); + // initialize data columns + for (int i = 0; i < columns.size(); i++) { + HiveColumnHandle column = columns.get(i); + checkState(column.getColumnType() == REGULAR, "column type must be regular"); + + types[i] = typeManager.getType(column.getTypeSignature()); + hiveTypes[i] = column.getHiveType(); + hiveIndexs[i] = column.getHiveColumnIndex(); + thriftIds[i] = getThriftIdWithFailOver(thriftFieldIdResolver, hiveIndexs[i]); + if (column.getFieldSet().isPresent()) { + thriftFieldIdGroupBuilder.put(thriftIds[i], HiveThriftFieldIdGroup.create(types[i], column.getFieldSet().get(), thriftFieldIdResolver)); + } + else { + thriftFieldIdGroupBuilder.put(thriftIds[i], new HiveThriftFieldIdGroup(ImmutableMap.of())); + } + } + + this.thriftFieldIdGroup = new HiveThriftFieldIdGroup(thriftFieldIdGroupBuilder.build()); + + // close immediately if the number of totalBytes is zero + if (totalBytes == 0) { + close(); + } + } + + @Override + public long getCompletedBytes() + { + if (!closed) { + updateCompletedBytes(); + } + return completedBytes; + } + + @Override + public long getReadTimeNanos() + { + return 0; + } + + private void updateCompletedBytes() + { + try { + long newCompletedBytes = (long) (totalBytes * recordReader.getProgress()); + completedBytes = min(totalBytes, max(completedBytes, newCompletedBytes)); + } + catch (IOException ignored) { + } + } + + @Override + public Type getType(int field) + { + return types[field]; + } + + @Override + public boolean advanceNextPosition() + { + try { + if (closed || !recordReader.next(key, value)) { + close(); + return false; + } + + // reset loaded flags + Arrays.fill(loaded, false); + + // decode value + rowData = deserializer.deserialize(value, thriftFieldIdGroup); + + return true; + } + catch (IOException | RuntimeException e) { + closeWithSuppression(this, e); + throw new PrestoException(HIVE_CURSOR_ERROR, + format("Failed to read split: %s %s:%s, total bytes: %s, completed bytes: %s", + path, start, start + totalBytes, totalBytes, completedBytes), + e); + } + } + + @Override + public boolean getBoolean(int fieldId) + { + checkState(!closed, "Cursor is closed"); + + validateType(fieldId, boolean.class); + if (!loaded[fieldId]) { + parseBooleanColumn(fieldId); + } + return booleans[fieldId]; + } + + private void parseBooleanColumn(int column) + { + loaded[column] = true; + + Object fieldValue = rowData.getFieldValueForThriftId(thriftIds[column]); + + if (fieldValue == null) { + nulls[column] = true; + } + else { + booleans[column] = (Boolean) fieldValue; + nulls[column] = false; + } + } + + @Override + public long getLong(int fieldId) + { + checkState(!closed, "Cursor is closed"); + + validateType(fieldId, long.class); + if (!loaded[fieldId]) { + parseLongColumn(fieldId); + } + return longs[fieldId]; + } + + private void parseLongColumn(int column) + { + loaded[column] = true; + + Object fieldValue = rowData.getFieldValueForThriftId(thriftIds[column]); + + if (fieldValue == null) { + nulls[column] = true; + } + else { + longs[column] = getLongExpressedValue(fieldValue, hiveStorageTimeZone); + nulls[column] = false; + } + } + + private static long getLongExpressedValue(Object value, DateTimeZone hiveTimeZone) + { + if (value instanceof Date) { + long storageTime = ((Date) value).getTime(); + // convert date from VM current time zone to UTC + long utcMillis = storageTime + DateTimeZone.getDefault().getOffset(storageTime); + return TimeUnit.MILLISECONDS.toDays(utcMillis); + } + if (value instanceof Timestamp) { + // The Hive SerDe parses timestamps using the default time zone of + // this JVM, but the data might have been written using a different + // time zone. We need to convert it to the configured time zone. + + // the timestamp that Hive parsed using the JVM time zone + long parsedJvmMillis = ((Timestamp) value).getTime(); + + // remove the JVM time zone correction from the timestamp + DateTimeZone jvmTimeZone = DateTimeZone.getDefault(); + long hiveMillis = jvmTimeZone.convertUTCToLocal(parsedJvmMillis); + + // convert to UTC using the real time zone for the underlying data + long utcMillis = hiveTimeZone.convertLocalToUTC(hiveMillis, false); + + return utcMillis; + } + if (value instanceof Float) { + return floatToRawIntBits(((Float) value)); + } + return ((Number) value).longValue(); + } + + @Override + public double getDouble(int fieldId) + { + checkState(!closed, "Cursor is closed"); + + validateType(fieldId, double.class); + if (!loaded[fieldId]) { + parseDoubleColumn(fieldId); + } + return doubles[fieldId]; + } + + private void parseDoubleColumn(int column) + { + loaded[column] = true; + + Object fieldValue = rowData.getFieldValueForThriftId(thriftIds[column]); + + if (fieldValue == null) { + nulls[column] = true; + } + else { + doubles[column] = ((Number) fieldValue).doubleValue(); + nulls[column] = false; + } + } + + @Override + public Slice getSlice(int fieldId) + { + checkState(!closed, "Cursor is closed"); + + validateType(fieldId, Slice.class); + if (!loaded[fieldId]) { + parseStringColumn(fieldId); + } + return slices[fieldId]; + } + + private void parseStringColumn(int column) + { + loaded[column] = true; + + Object fieldValue = rowData.getFieldValueForThriftId(thriftIds[column]); + + if (fieldValue == null) { + nulls[column] = true; + } + else { + slices[column] = getSliceExpressedValue(fieldValue, types[column]); + nulls[column] = false; + } + } + + private static Slice getSliceExpressedValue(Object value, Type type) + { + Slice sliceValue; + if (value instanceof String) { + sliceValue = Slices.utf8Slice((String) value); + } + else if (value instanceof byte[]) { + sliceValue = Slices.wrappedBuffer((byte[]) value); + } + else if (value instanceof HiveVarchar) { + sliceValue = Slices.utf8Slice(((HiveVarchar) value).getValue()); + } + else if (value instanceof HiveChar) { + sliceValue = Slices.utf8Slice(((HiveChar) value).getValue()); + } + else if (value instanceof Integer) { + sliceValue = Slices.utf8Slice(value.toString()); + } + else { + throw new IllegalStateException("unsupported string field type: " + value.getClass().getName()); + } + if (isVarcharType(type)) { + sliceValue = truncateToLength(sliceValue, type); + } + if (isCharType(type)) { + sliceValue = truncateToLengthAndTrimSpaces(sliceValue, type); + } + + return sliceValue; + } + + private void parseDecimalColumn(int column) + { + loaded[column] = true; + + Object fieldValue = rowData.getFieldValueForThriftId(thriftIds[column]); + + if (fieldValue == null) { + nulls[column] = true; + } + else { + HiveDecimal decimal = (HiveDecimal) fieldValue; + DecimalType columnType = (DecimalType) types[column]; + BigInteger unscaledDecimal = rescale(decimal.unscaledValue(), decimal.scale(), columnType.getScale()); + + if (columnType.isShort()) { + longs[column] = unscaledDecimal.longValue(); + } + else { + slices[column] = Decimals.encodeUnscaledValue(unscaledDecimal); + } + nulls[column] = false; + } + } + + @Override + public Object getObject(int fieldId) + { + checkState(!closed, "Cursor is closed"); + + validateType(fieldId, Block.class); + if (!loaded[fieldId]) { + parseObjectColumn(fieldId); + } + return objects[fieldId]; + } + + private void parseObjectColumn(int column) + { + loaded[column] = true; + + Object fieldValue = rowData.getFieldValueForThriftId(thriftIds[column]); + + if (fieldValue == null) { + nulls[column] = true; + } + else { + ThriftFieldIdResolver resolver = thriftFieldIdResolver.getNestedResolver(hiveIndexs[column]); + objects[column] = getBlockObject(types[column], resolver, fieldValue, hiveStorageTimeZone); + nulls[column] = false; + } + } + + @Override + public boolean isNull(int fieldId) + { + checkState(!closed, "Cursor is closed"); + + if (!loaded[fieldId]) { + parseColumn(fieldId); + } + return nulls[fieldId]; + } + + private void parseColumn(int column) + { + Type type = types[column]; + if (BOOLEAN.equals(type)) { + parseBooleanColumn(column); + } + else if (BIGINT.equals(type)) { + parseLongColumn(column); + } + else if (INTEGER.equals(type)) { + parseLongColumn(column); + } + else if (SMALLINT.equals(type)) { + parseLongColumn(column); + } + else if (TINYINT.equals(type)) { + parseLongColumn(column); + } + else if (REAL.equals(type)) { + parseLongColumn(column); + } + else if (DOUBLE.equals(type)) { + parseDoubleColumn(column); + } + else if (isVarcharType(type) || VARBINARY.equals(type)) { + parseStringColumn(column); + } + else if (isCharType(type)) { + parseStringColumn(column); + } + else if (isStructuralType(hiveTypes[column])) { + parseObjectColumn(column); + } + else if (DATE.equals(type)) { + parseLongColumn(column); + } + else if (TIMESTAMP.equals(type)) { + parseLongColumn(column); + } + else if (type instanceof DecimalType) { + parseDecimalColumn(column); + } + else { + throw new UnsupportedOperationException("Unsupported column type: " + type); + } + } + + private void validateType(int fieldId, Class type) + { + if (!types[fieldId].getJavaType().equals(type)) { + // we don't use Preconditions.checkArgument because it requires boxing fieldId, which affects inner loop performance + throw new IllegalArgumentException(String.format("Expected field to be %s, actual %s (field %s)", type, types[fieldId], fieldId)); + } + } + + @Override + public void close() + { + // some hive input formats are broken and bad things can happen if you close them multiple times + if (closed) { + return; + } + closed = true; + + updateCompletedBytes(); + + try { + recordReader.close(); + } + catch (IOException e) { + throw new RuntimeException("Error closing thrift record reader", e); + } + } + + private static Block getBlockObject(Type type, ThriftFieldIdResolver resolver, Object object, DateTimeZone hiveStorageTimeZone) + { + return requireNonNull(serializeObject(type, resolver, null, object, hiveStorageTimeZone), "serialized result is null"); + } + + private static Block serializeObject(Type type, ThriftFieldIdResolver resolver, BlockBuilder builder, Object object, DateTimeZone hiveStorageTimeZone) + { + if (object == null) { + requireNonNull(builder, "parent builder is null").appendNull(); + return null; + } + if (!isStructuralType(type)) { + serializePrimitive(type, resolver, builder, object, hiveStorageTimeZone); + return null; + } + else if (isArrayType(type)) { + return serializeList(type, resolver, builder, object, hiveStorageTimeZone); + } + else if (isMapType(type)) { + return serializeMap(type, resolver, builder, object, hiveStorageTimeZone); + } + else if (isRowType(type)) { + return serializeStruct(type, resolver, builder, object, hiveStorageTimeZone); + } + throw new RuntimeException("Unknown object type: " + type); + } + + private static Block serializeList(Type type, ThriftFieldIdResolver resolver, BlockBuilder builder, Object object, DateTimeZone hiveStorageTimeZone) + { + List list = (List) requireNonNull(object, "object is null"); + List typeParameters = type.getTypeParameters(); + checkArgument(typeParameters.size() == 1, "list must have exactly 1 type parameter"); + Type elementType = typeParameters.get(0); + ThriftFieldIdResolver elementResolver = resolver.getNestedResolver(0); + BlockBuilder currentBuilder; + if (builder != null) { + currentBuilder = builder.beginBlockEntry(); + } + else { + currentBuilder = elementType.createBlockBuilder(null, list.size()); + } + + for (Object element : list) { + serializeObject(elementType, elementResolver, currentBuilder, element, hiveStorageTimeZone); + } + + if (builder != null) { + builder.closeEntry(); + return null; + } + else { + Block resultBlock = currentBuilder.build(); + return resultBlock; + } + } + + private static Block serializeMap(Type type, ThriftFieldIdResolver resolver, BlockBuilder builder, Object object, DateTimeZone hiveStorageTimeZone) + { + Map map = (Map) requireNonNull(object, "object is null"); + List typeParameters = type.getTypeParameters(); + checkArgument(typeParameters.size() == 2, "map must have exactly 2 type parameter"); + Type keyType = typeParameters.get(0); + Type valueType = typeParameters.get(1); + ThriftFieldIdResolver keyResolver = resolver.getNestedResolver(0); + ThriftFieldIdResolver valueResolver = resolver.getNestedResolver(1); + boolean builderSynthesized = false; + if (builder == null) { + builderSynthesized = true; + builder = type.createBlockBuilder(null, 1); + } + BlockBuilder currentBuilder = builder.beginBlockEntry(); + + for (Map.Entry entry : map.entrySet()) { + // Hive skips map entries with null keys + if (entry.getKey() != null) { + serializeObject(keyType, keyResolver, currentBuilder, entry.getKey(), hiveStorageTimeZone); + serializeObject(valueType, valueResolver, currentBuilder, entry.getValue(), hiveStorageTimeZone); + } + } + + builder.closeEntry(); + if (builderSynthesized) { + return (Block) type.getObject(builder, 0); + } + else { + return null; + } + } + + private static Block serializeStruct(Type type, ThriftFieldIdResolver resolver, BlockBuilder builder, Object object, DateTimeZone hiveStorageTimeZone) + { + ThriftGenericRow structData = (ThriftGenericRow) requireNonNull(object, "object is null"); + List typeParameters = type.getTypeParameters(); + + boolean builderSynthesized = false; + if (builder == null) { + builderSynthesized = true; + builder = type.createBlockBuilder(null, 1); + } + BlockBuilder currentBuilder = builder.beginBlockEntry(); + + for (int i = 0; i < typeParameters.size(); i++) { + Object fieldValue = structData.getFieldValueForThriftId(getThriftIdWithFailOver(resolver, i)); + if (fieldValue == null) { + currentBuilder.appendNull(); + } + else { + serializeObject(typeParameters.get(i), resolver.getNestedResolver(i), currentBuilder, fieldValue, hiveStorageTimeZone); + } + } + + builder.closeEntry(); + if (builderSynthesized) { + return (Block) type.getObject(builder, 0); + } + else { + return null; + } + } + + private static void serializePrimitive(Type type, ThriftFieldIdResolver resolver, BlockBuilder builder, Object object, DateTimeZone hiveStorageTimeZone) + { + requireNonNull(builder, "parent builder is null"); + requireNonNull(object, "object is null"); + + if (BOOLEAN.equals(type)) { + BOOLEAN.writeBoolean(builder, (Boolean) object); + } + else if (BIGINT.equals(type)) { + BIGINT.writeLong(builder, getLongExpressedValue(object, hiveStorageTimeZone)); + } + else if (INTEGER.equals(type)) { + INTEGER.writeLong(builder, getLongExpressedValue(object, hiveStorageTimeZone)); + } + else if (SMALLINT.equals(type)) { + SMALLINT.writeLong(builder, getLongExpressedValue(object, hiveStorageTimeZone)); + } + else if (TINYINT.equals(type)) { + TINYINT.writeLong(builder, getLongExpressedValue(object, hiveStorageTimeZone)); + } + else if (REAL.equals(type)) { + REAL.writeLong(builder, getLongExpressedValue(object, hiveStorageTimeZone)); + } + else if (DOUBLE.equals(type)) { + DOUBLE.writeDouble(builder, ((Number) object).doubleValue()); + } + else if (isVarcharType(type) || VARBINARY.equals(type) || isCharType(type)) { + type.writeSlice(builder, getSliceExpressedValue(object, type)); + } + else if (DATE.equals(type)) { + DATE.writeLong(builder, getLongExpressedValue(object, hiveStorageTimeZone)); + } + else if (TIMESTAMP.equals(type)) { + TIMESTAMP.writeLong(builder, getLongExpressedValue(object, hiveStorageTimeZone)); + } + else if (type instanceof DecimalType) { + HiveDecimal decimal = (HiveDecimal) object; + DecimalType decimalType = (DecimalType) type; + BigInteger unscaledDecimal = rescale(decimal.unscaledValue(), decimal.scale(), decimalType.getScale()); + if (decimalType.isShort()) { + decimalType.writeLong(builder, unscaledDecimal.longValue()); + } + else { + decimalType.writeSlice(builder, Decimals.encodeUnscaledValue(unscaledDecimal)); + } + } + else { + throw new UnsupportedOperationException("Unsupported primitive type: " + type); + } + } + + private static short getThriftIdWithFailOver(ThriftFieldIdResolver thriftFieldIdResolver, int hiveIndex) + { + try { + return thriftFieldIdResolver.getThriftId(hiveIndex); + } + catch (PrestoException e) { + return NON_EXISTED_THRIFT_ID; + } + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftHiveRecordCursorProvider.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftHiveRecordCursorProvider.java new file mode 100644 index 0000000000000..8c10e78f5f332 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/thrift/ThriftHiveRecordCursorProvider.java @@ -0,0 +1,163 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive.thrift; + +import com.facebook.presto.hive.HdfsEnvironment; +import com.facebook.presto.hive.HiveColumnHandle; +import com.facebook.presto.hive.HiveRecordCursorProvider; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.RecordCursor; +import com.facebook.presto.spi.predicate.TupleDomain; +import com.facebook.presto.spi.type.TypeManager; +import com.google.common.collect.ImmutableSet; +import com.hadoop.compression.lzo.LzoIndex; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapred.RecordReader; +import org.joda.time.DateTimeZone; + +import javax.inject.Inject; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; + +import static com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_METADATA; +import static com.facebook.presto.hive.HiveStorageFormat.THRIFTBINARY; +import static com.facebook.presto.hive.HiveUtil.checkCondition; +import static com.facebook.presto.hive.HiveUtil.createRecordReader; +import static com.facebook.presto.hive.HiveUtil.getDeserializerClassName; +import static com.facebook.presto.hive.HiveUtil.getLzopIndexPath; +import static com.facebook.presto.hive.HiveUtil.isLzopCompressedFile; +import static java.util.Objects.requireNonNull; +import static org.apache.hadoop.hive.serde.Constants.SERIALIZATION_CLASS; + +public class ThriftHiveRecordCursorProvider + implements HiveRecordCursorProvider +{ + private static final String THRIFT_GENERIC_ROW = ThriftGenericRow.class.getName(); + private static final Set THRIFT_SERDE_CLASS_NAMES = ImmutableSet.builder() + .add(ThriftGeneralDeserializer.class.getName()) + .add(THRIFTBINARY.getSerDe()) + .build(); + private final HdfsEnvironment hdfsEnvironment; + private final ThriftFieldIdResolverFactory thriftFieldIdResolverFactory; + + @Inject + public ThriftHiveRecordCursorProvider(HdfsEnvironment hdfsEnvironment, ThriftFieldIdResolverFactory thriftFieldIdResolverFactory) + { + this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); + this.thriftFieldIdResolverFactory = requireNonNull(thriftFieldIdResolverFactory, "thriftFieldIdResolverFactory is null"); + } + + @Override + public Optional createRecordCursor( + Configuration configuration, + ConnectorSession session, + Path path, + long start, + long length, + long fileSize, + Properties schema, + List columns, + TupleDomain effectivePredicate, + DateTimeZone hiveStorageTimeZone, + TypeManager typeManager) + { + if (!THRIFT_SERDE_CLASS_NAMES.contains(getDeserializerClassName(schema))) { + return Optional.empty(); + } + + // We only allow the table which specified its serialization class is compatible to + // our thrift general row, if the SerDe is LazyBinarySerDe. + if (THRIFTBINARY.getSerDe().equals(getDeserializerClassName(schema)) && !THRIFT_GENERIC_ROW.equals(getSerializationClassName(schema))) { + return Optional.empty(); + } + + setPropertyIfUnset(schema, "elephantbird.mapred.input.bad.record.check.only.in.close", Boolean.toString(false)); + setPropertyIfUnset(schema, "elephantbird.mapred.input.bad.record.threshold", Float.toString(0.0f)); + + // re-align split range + if (isLzopCompressedFile(path)) { + LzoIndex index = new LzoIndex(); + try { + index = LzoIndex.readIndex(hdfsEnvironment.getFileSystem(session.getUser(), getLzopIndexPath(path), configuration), path); + } + catch (IOException ignored) { + // ignored + } + + // re-align split start + if (index.isEmpty() && start != 0) { + // empty index and split not start from beginning + start = LzoIndex.NOT_FOUND; + } + if (!index.isEmpty()) { + // align start based on index + start = index.alignSliceStartToIndex(start, start + length); + } + + // re-align split end + if (start == LzoIndex.NOT_FOUND) { + // split start cannot be find, then we should skip this split + start = 0; + length = 0; + } + else { + // if index is empty but start is not NOT_FOUND, read the whole file, otherwise align split end. + length = index.isEmpty() ? fileSize : + (Math.min(index.alignSliceEndToIndex(start + length, fileSize), fileSize) - start); + } + } + + long finalStart = start; + long finalLength = length; + RecordReader recordReader = hdfsEnvironment.doAs(session.getUser(), + () -> createRecordReader(configuration, path, finalStart, finalLength, schema, columns)); + + return Optional.of(new ThriftHiveRecordCursor<>( + genericRecordReader(recordReader), + path, + start, + length, + schema, + columns, + hiveStorageTimeZone, + typeManager, + thriftFieldIdResolverFactory.createResolver(schema))); + } + + @SuppressWarnings("unchecked") + private static RecordReader genericRecordReader(RecordReader recordReader) + { + return (RecordReader) recordReader; + } + + private static void setPropertyIfUnset(Properties schema, String key, String value) + { + if (schema.getProperty(key) == null) { + schema.setProperty(key, value); + } + } + + private static String getSerializationClassName(Properties schema) + { + String name = schema.getProperty(SERIALIZATION_CLASS); + checkCondition(name != null, HIVE_INVALID_METADATA, "Table or partition is missing Hive property: %s", SERIALIZATION_CLASS); + return name; + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java new file mode 100644 index 0000000000000..f823576d83894 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/PooledTTransportFactory.java @@ -0,0 +1,261 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive.util; + +import com.facebook.presto.hive.authentication.HiveMetastoreAuthentication; +import com.google.common.net.HostAndPort; +import org.apache.commons.pool2.BasePooledObjectFactory; +import org.apache.commons.pool2.PooledObject; +import org.apache.commons.pool2.impl.DefaultPooledObject; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Proxy; +import java.net.Socket; +import java.net.SocketAddress; +import java.net.SocketException; + +import static java.util.Objects.requireNonNull; + +public class PooledTTransportFactory + extends BasePooledObjectFactory +{ + private final TTransportPool pool; + private final String host; + private final int port; + private final HostAndPort socksProxy; + private final int timeoutMillis; + private final HiveMetastoreAuthentication metastoreAuthentication; + + public PooledTTransportFactory(TTransportPool pool, + String host, + int port, + @Nullable HostAndPort socksProxy, + int timeoutMillis, + HiveMetastoreAuthentication metastoreAuthentication) + { + this.pool = requireNonNull(pool, "pool is null"); + this.host = requireNonNull(host, "host is null"); + this.port = port; + this.socksProxy = socksProxy; + this.timeoutMillis = timeoutMillis; + this.metastoreAuthentication = requireNonNull(metastoreAuthentication, "metastoreAuthentication is null"); + } + + @Override + public void activateObject(PooledObject pooledObject) + throws Exception + { + pooledObject.getObject().flush(); + } + + @Override + public boolean validateObject(PooledObject pooledObject) + { + try { + return (pooledObject.getObject().isOpen() && + ((PooledTTransport) pooledObject.getObject()).isReachable(timeoutMillis)); + } + catch (Exception e) { + return false; + } + } + + @Override + public TTransport create() + throws Exception + { + TTransport transport; + if (socksProxy == null) { + transport = new TSocket(host, port, timeoutMillis); + } + else { + SocketAddress address = InetSocketAddress.createUnresolved(socksProxy.getHost(), + socksProxy.getPort()); + Socket socket = new Socket(new Proxy(Proxy.Type.SOCKS, address)); + try { + socket.connect(InetSocketAddress.createUnresolved(host, port), timeoutMillis); + socket.setSoTimeout(timeoutMillis); + transport = new TSocket(socket); + } + catch (SocketException e) { + if (socket.isConnected()) { + try { + socket.close(); + } + catch (IOException ioException) { + // ignored + } + } + throw e; + } + } + TTransport authenticatedTransport = metastoreAuthentication.authenticate(transport, host); + if (!authenticatedTransport.isOpen()) { + authenticatedTransport.open(); + } + + return new PooledTTransport(authenticatedTransport, pool, + HostAndPort.fromParts(host, port).toString()); + } + + @Override + public void destroyObject(PooledObject pooledObject) + { + try { + ((PooledTTransport) pooledObject.getObject()).getTTransport().close(); + } + catch (ClassCastException e) { + // ignore + } + pooledObject.invalidate(); + } + + @Override + public PooledObject wrap(TTransport transport) + { + return new DefaultPooledObject(transport); + } + + @Override + public void passivateObject(PooledObject pooledObject) + { + try { + pooledObject.getObject().flush(); + } + catch (TTransportException e) { + destroyObject(pooledObject); + } + } + + private static class PooledTTransport + extends TTransport + { + private final String remote; + private final TTransportPool pool; + private final TTransport transport; + + public PooledTTransport(TTransport transport, TTransportPool pool, String remote) + { + this.transport = transport; + this.pool = pool; + this.remote = remote; + } + + public TTransport getTTransport() + { + return transport; + } + + public boolean isReachable(int timeoutMillis) + throws ClassCastException, IOException + { + return ((TSocket) transport).getSocket().getInetAddress().isReachable(timeoutMillis); + } + + @Override + public void close() + { + try { + pool.returnObject(remote, this, transport); + } + catch (Exception e) { + transport.close(); + } + } + + @Override + public boolean isOpen() + { + return transport.isOpen(); + } + + @Override + public boolean peek() + { + return transport.peek(); + } + + @Override + public byte[] getBuffer() + { + return transport.getBuffer(); + } + + @Override + public int getBufferPosition() + { + return transport.getBufferPosition(); + } + + @Override + public int getBytesRemainingInBuffer() + { + return transport.getBytesRemainingInBuffer(); + } + + @Override + public void consumeBuffer(int len) + { + transport.consumeBuffer(len); + } + + @Override + public void open() + throws TTransportException + { + transport.open(); + } + + @Override + public int readAll(byte[] bytes, int off, int len) + throws TTransportException + { + return transport.readAll(bytes, off, len); + } + + @Override + public int read(byte[] bytes, int off, int len) + throws TTransportException + { + return transport.read(bytes, off, len); + } + + @Override + public void write(byte[] bytes) + throws TTransportException + { + transport.write(bytes); + } + + @Override + public void write(byte[] bytes, int off, int len) + throws TTransportException + { + transport.write(bytes, off, len); + } + + @Override + public void flush() + throws TTransportException + { + transport.flush(); + } + } +} diff --git a/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java new file mode 100644 index 0000000000000..7d2a3b5af23b1 --- /dev/null +++ b/presto-hive/src/main/java/com/facebook/presto/twitter/hive/util/TTransportPool.java @@ -0,0 +1,93 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive.util; + +import com.google.common.net.HostAndPort; +import org.apache.commons.pool2.ObjectPool; +import org.apache.commons.pool2.PooledObjectFactory; +import org.apache.commons.pool2.impl.GenericObjectPool; +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; +import org.apache.thrift.transport.TTransport; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class TTransportPool +{ + private final ConcurrentMap> pools = new ConcurrentHashMap(); + private GenericObjectPoolConfig poolConfig; + + public TTransportPool(GenericObjectPoolConfig poolConfig) + { + this.poolConfig = poolConfig; + } + + protected synchronized void add(String remote, PooledObjectFactory transportFactory) + { + pools.putIfAbsent(remote, new GenericObjectPool(transportFactory, poolConfig)); + } + + protected TTransport get(String remote, PooledObjectFactory transportFactory) + throws Exception + { + add(remote, transportFactory); + return get(remote); + } + + protected TTransport get(String remote) + throws Exception + { + ObjectPool pool = pools.get(remote); + if (pool == null) { + return null; + } + return pool.borrowObject(); + } + + public TTransport borrowObject(String host, int port, PooledObjectFactory transportFactory) + throws Exception + { + return get(HostAndPort.fromParts(host, port).toString(), transportFactory); + } + + public TTransport borrowObject(String host, int port) + throws Exception + { + return get(HostAndPort.fromParts(host, port).toString()); + } + + public void returnObject(String remote, TTransport pooledTransport, TTransport transport) + { + if (remote == null) { + transport.close(); + return; + } + ObjectPool pool = pools.get(remote); + if (pool == null) { + transport.close(); + return; + } + try { + pool.returnObject(pooledTransport); + } + catch (Exception e) { + transport.close(); + } + } + + public void returnObject(TTransport transport) + { + transport.close(); + } +} diff --git a/presto-hive/src/main/java/parquet/io/ColumnIOConverter.java b/presto-hive/src/main/java/parquet/io/ColumnIOConverter.java index ca05ba94e8b1c..e657878e54f35 100644 --- a/presto-hive/src/main/java/parquet/io/ColumnIOConverter.java +++ b/presto-hive/src/main/java/parquet/io/ColumnIOConverter.java @@ -27,9 +27,9 @@ import java.util.Locale; import java.util.Optional; +import static com.facebook.presto.hive.parquet.ParquetTypeUtils.findColumnIObyName; import static com.facebook.presto.hive.parquet.ParquetTypeUtils.getArrayElementColumn; import static com.facebook.presto.hive.parquet.ParquetTypeUtils.getMapKeyValueColumn; -import static com.facebook.presto.hive.parquet.ParquetTypeUtils.lookupColumnByName; import static com.facebook.presto.spi.type.StandardTypes.ARRAY; import static com.facebook.presto.spi.type.StandardTypes.MAP; import static com.facebook.presto.spi.type.StandardTypes.ROW; @@ -61,7 +61,7 @@ public static Optional constructField(Type type, ColumnIO columnIO) for (int i = 0; i < fields.size(); i++) { NamedTypeSignature namedTypeSignature = fields.get(i).getNamedTypeSignature(); String name = namedTypeSignature.getName().get().toLowerCase(Locale.ENGLISH); - Optional field = constructField(parameters.get(i), lookupColumnByName(groupColumnIO, name)); + Optional field = constructField(parameters.get(i), findColumnIObyName(groupColumnIO, name)); structHasParameters |= field.isPresent(); fieldsBuilder.add(field); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java index 7d214b0e2f941..0e1d6da59b2ee 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java @@ -162,6 +162,7 @@ import static com.facebook.presto.hive.HiveStorageFormat.RCTEXT; import static com.facebook.presto.hive.HiveStorageFormat.SEQUENCEFILE; import static com.facebook.presto.hive.HiveStorageFormat.TEXTFILE; +import static com.facebook.presto.hive.HiveStorageFormat.THRIFTBINARY; import static com.facebook.presto.hive.HiveTableProperties.BUCKETED_BY_PROPERTY; import static com.facebook.presto.hive.HiveTableProperties.BUCKET_COUNT_PROPERTY; import static com.facebook.presto.hive.HiveTableProperties.PARTITIONED_BY_PROPERTY; @@ -428,7 +429,7 @@ private static RowType toRowType(List columns) }).collect(toList())) .build(); - protected Set createTableFormats = difference(ImmutableSet.copyOf(HiveStorageFormat.values()), ImmutableSet.of(AVRO)); + protected Set createTableFormats = difference(ImmutableSet.copyOf(HiveStorageFormat.values()), ImmutableSet.of(AVRO, THRIFTBINARY)); private static final JoinCompiler JOIN_COMPILER = new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig()); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java index 591b7269cef2c..a0bdfd1f9932e 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java @@ -82,6 +82,7 @@ public void testDefaults() .setUseOrcColumnNames(false) .setParquetPredicatePushdownEnabled(true) .setParquetOptimizedReaderEnabled(true) + .setParquetNestedFieldsProjectionPushdownEnabled(false) .setAssumeCanonicalPartitionKeys(false) .setOrcBloomFiltersEnabled(false) .setOrcDefaultBloomFilterFpp(0.05) @@ -158,6 +159,7 @@ public void testExplicitPropertyMappings() .put("hive.orc.use-column-names", "true") .put("hive.parquet-predicate-pushdown.enabled", "false") .put("hive.parquet-optimized-reader.enabled", "false") + .put("hive.parquet-nested-fields-projection-pushdown.enabled", "true") .put("hive.orc.bloom-filters.enabled", "true") .put("hive.orc.default-bloom-filter-fpp", "0.96") .put("hive.orc.max-merge-distance", "22kB") @@ -230,6 +232,7 @@ public void testExplicitPropertyMappings() .setUseOrcColumnNames(true) .setParquetPredicatePushdownEnabled(false) .setParquetOptimizedReaderEnabled(false) + .setParquetNestedFieldsProjectionPushdownEnabled(true) .setAssumeCanonicalPartitionKeys(true) .setOrcBloomFiltersEnabled(true) .setOrcDefaultBloomFilterFpp(0.96) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveFileFormats.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveFileFormats.java index 14bb386c75c86..6b6751e8053d8 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveFileFormats.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveFileFormats.java @@ -28,6 +28,9 @@ import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.RowType; import com.facebook.presto.testing.TestingConnectorSession; +import com.facebook.presto.twitter.hive.thrift.HiveThriftFieldIdResolverFactory; +import com.facebook.presto.twitter.hive.thrift.ThriftGenericRow; +import com.facebook.presto.twitter.hive.thrift.ThriftHiveRecordCursorProvider; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -70,6 +73,7 @@ import static com.facebook.presto.hive.HiveStorageFormat.RCTEXT; import static com.facebook.presto.hive.HiveStorageFormat.SEQUENCEFILE; import static com.facebook.presto.hive.HiveStorageFormat.TEXTFILE; +import static com.facebook.presto.hive.HiveStorageFormat.THRIFTBINARY; import static com.facebook.presto.hive.HiveTestUtils.HDFS_ENVIRONMENT; import static com.facebook.presto.hive.HiveTestUtils.SESSION; import static com.facebook.presto.hive.HiveTestUtils.TYPE_MANAGER; @@ -86,6 +90,7 @@ import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toList; import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_INPUT_FORMAT; +import static org.apache.hadoop.hive.serde.Constants.SERIALIZATION_CLASS; import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB; import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardListObjectInspector; import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardStructObjectInspector; @@ -497,6 +502,43 @@ public void testParquetThrift(int rowCount) testCursorProvider(cursorProvider, split, PARQUET, testColumns, SESSION, 1); } + public void testLZOThrift() + throws Exception + { + RowType nameType = RowType.anonymous(ImmutableList.of(createUnboundedVarcharType(), createUnboundedVarcharType())); + RowType phoneType = RowType.anonymous(ImmutableList.of(createUnboundedVarcharType(), createUnboundedVarcharType())); + RowType personType = RowType.anonymous(ImmutableList.of(nameType, INTEGER, createUnboundedVarcharType(), new ArrayType(phoneType))); + + List testColumns = ImmutableList.of( + new TestColumn( + "persons", + getStandardListObjectInspector( + getStandardStructObjectInspector( + ImmutableList.of("name", "id", "email", "phones"), + ImmutableList.of( + getStandardStructObjectInspector( + ImmutableList.of("first_name", "last_name"), + ImmutableList.of(javaStringObjectInspector, javaStringObjectInspector)), + javaIntObjectInspector, + javaStringObjectInspector, + getStandardListObjectInspector( + getStandardStructObjectInspector( + ImmutableList.of("number", "type"), + ImmutableList.of(javaStringObjectInspector, javaStringObjectInspector)))))), + null, + arrayBlockOf(personType, + rowBlockOf(ImmutableList.of(nameType, INTEGER, createUnboundedVarcharType(), new ArrayType(phoneType)), + rowBlockOf(ImmutableList.of(createUnboundedVarcharType(), createUnboundedVarcharType()), "Bob", "Roberts"), + 0, + "bob.roberts@example.com", + arrayBlockOf(phoneType, rowBlockOf(ImmutableList.of(createUnboundedVarcharType(), createUnboundedVarcharType()), "1234567890", null)))))); + + File file = new File(this.getClass().getClassLoader().getResource("addressbook.thrift.lzo").getPath()); + FileSplit split = new FileSplit(new Path(file.getAbsolutePath()), 0, file.length(), new String[0]); + HiveRecordCursorProvider cursorProvider = new ThriftHiveRecordCursorProvider(HDFS_ENVIRONMENT, new HiveThriftFieldIdResolverFactory()); + testCursorProvider(cursorProvider, split, THRIFTBINARY, testColumns, SESSION, 1); + } + @Test(dataProvider = "rowCount") public void testDwrf(int rowCount) throws Exception @@ -664,7 +706,9 @@ private void testCursorProvider(HiveRecordCursorProvider cursorProvider, splitProperties.setProperty(SERIALIZATION_LIB, storageFormat.getSerDe()); splitProperties.setProperty("columns", Joiner.on(',').join(transform(filter(testColumns, not(TestColumn::isPartitionKey)), TestColumn::getName))); splitProperties.setProperty("columns.types", Joiner.on(',').join(transform(filter(testColumns, not(TestColumn::isPartitionKey)), TestColumn::getType))); - + if (storageFormat.equals(THRIFTBINARY)) { + splitProperties.setProperty(SERIALIZATION_CLASS, ThriftGenericRow.class.getName()); + } List partitionKeys = testColumns.stream() .filter(TestColumn::isPartitionKey) .map(input -> new HivePartitionKey(input.getName(), (String) input.getWriteValue())) @@ -672,6 +716,9 @@ private void testCursorProvider(HiveRecordCursorProvider cursorProvider, Configuration configuration = new Configuration(); configuration.set("io.compression.codecs", LzoCodec.class.getName() + "," + LzopCodec.class.getName()); + if (storageFormat.equals(THRIFTBINARY)) { + configuration.set("io.compression.codecs", "com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec"); + } Optional pageSource = HivePageSourceProvider.createHivePageSource( ImmutableSet.of(cursorProvider), ImmutableSet.of(), diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java index 0a1141e9a4a89..45a511c9f95fd 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveIntegrationSmokeTest.java @@ -2789,6 +2789,9 @@ private List getAllTestingHiveStorageFormat() Session session = getSession(); ImmutableList.Builder formats = ImmutableList.builder(); for (HiveStorageFormat hiveStorageFormat : HiveStorageFormat.values()) { + if (hiveStorageFormat.equals(HiveStorageFormat.THRIFTBINARY)) { + continue; + } formats.add(new TestingHiveStorageFormat(session, hiveStorageFormat)); } formats.add(new TestingHiveStorageFormat( diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePageSink.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePageSink.java index 002bbdf39a6f5..801a0ac15ed9c 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePageSink.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHivePageSink.java @@ -101,6 +101,9 @@ public void testAllFormats() try { ExtendedHiveMetastore metastore = createTestingFileHiveMetastore(new File(tempDir, "metastore")); for (HiveStorageFormat format : HiveStorageFormat.values()) { + if (format.equals(HiveStorageFormat.THRIFTBINARY)) { + continue; + } config.setHiveStorageFormat(format); config.setHiveCompressionCodec(NONE); long uncompressedLength = writeTestFile(config, metastore, makeFileName(tempDir, config)); diff --git a/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperMetastoreMonitor.java b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperMetastoreMonitor.java new file mode 100644 index 0000000000000..1f89464aedce3 --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperMetastoreMonitor.java @@ -0,0 +1,157 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive; + +import com.facebook.presto.twitter.hive.util.TestUtils; +import com.google.common.collect.ImmutableList; +import com.google.common.net.HostAndPort; +import io.airlift.log.Logger; +import org.I0Itec.zkclient.ZkClient; +import org.I0Itec.zkclient.exception.ZkMarshallingError; +import org.I0Itec.zkclient.serialize.ZkSerializer; +import org.apache.curator.test.TestingServer; +import org.json.simple.JSONObject; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.testng.Assert.assertTrue; + +public class TestZookeeperMetastoreMonitor +{ + private static final Logger log = Logger.get(TestZookeeperMetastoreMonitor.class); + + private ZookeeperMetastoreMonitor zkMetastoreMonitor; + private TestingServer zkServer; + private ZkClient zkClient; + private final String zkPath = "/metastores"; + + public TestZookeeperMetastoreMonitor() + throws Exception + { + zkServer = new TestingServer(TestUtils.findUnusedPort()); + zkClient = new ZkClient(zkServer.getConnectString(), 30_000, 30_000); + + // Set the serializer + zkClient.setZkSerializer(new ZkSerializer() { + @Override + public byte[] serialize(Object o) throws ZkMarshallingError + { + try { + return o.toString().getBytes(StandardCharsets.UTF_8); + } + catch (Exception e) { + log.warn("Exception in serializing " + e); + } + return "".getBytes(); + } + + @Override + public Object deserialize(byte[] bytes) throws ZkMarshallingError + { + return null; + } + }); + } + + @AfterClass + public void destroy() + throws IOException + { + zkMetastoreMonitor.close(); + zkClient.close(); + zkServer.close(); + } + + @BeforeTest + public void setUp() + throws Exception + { + log.info("Cleaning up zookeeper"); + zkClient.getChildren("/").stream() + .filter(child -> !child.equals("zookeeper")) + .forEach(child -> zkClient.deleteRecursive("/" + child)); + + zkClient.unsubscribeAll(); + + zkClient.createPersistent(zkPath); + zkMetastoreMonitor = new ZookeeperMetastoreMonitor(zkServer.getConnectString(), zkPath, 3, 500); + } + + @Test + public void testGetServers() throws Exception + { + List servers; + List expected; + assertTrue(zkMetastoreMonitor.getServers().isEmpty()); + + addServerToZk("nameNode1", "host1", 10001); + // Sleep for some time so that event can be propagated. + TimeUnit.MILLISECONDS.sleep(100); + servers = zkMetastoreMonitor.getServers(); + expected = ImmutableList.of(HostAndPort.fromParts("host1", 10001)); + assertTrue(servers.containsAll(expected) && expected.containsAll(servers)); + + addServerToZk("nameNode2", "host2", 10002); + // Sleep for some time so that event can be propagated. + TimeUnit.MILLISECONDS.sleep(100); + servers = zkMetastoreMonitor.getServers(); + expected = ImmutableList.of(HostAndPort.fromParts("host1", 10001), HostAndPort.fromParts("host2", 10002)); + assertTrue(servers.containsAll(expected) && expected.containsAll(servers)); + + // Change value of an existing name node + addServerToZk("nameNode2", "host2", 10003); + // Sleep for some time so that event can be propagated. + TimeUnit.MILLISECONDS.sleep(100); + servers = zkMetastoreMonitor.getServers(); + expected = ImmutableList.of(HostAndPort.fromParts("host1", 10001), HostAndPort.fromParts("host2", 10003)); + assertTrue(servers.containsAll(expected) && expected.containsAll(servers)); + + // Delete an existing name node + zkClient.delete(getPathForNameNode("nameNode1")); + // Sleep for some time so that event can be propagated. + TimeUnit.MILLISECONDS.sleep(100); + servers = zkMetastoreMonitor.getServers(); + expected = ImmutableList.of(HostAndPort.fromParts("host2", 10003)); + assertTrue(servers.containsAll(expected) && expected.containsAll(servers), servers.toString()); + } + + private void addServerToZk(String nameNode, String host, int port) + { + JSONObject serviceEndpoint = new JSONObject(); + serviceEndpoint.put("host", host); + serviceEndpoint.put("port", port); + JSONObject jsonObject = new JSONObject(); + jsonObject.put("serviceEndpoint", serviceEndpoint); + + String path = getPathForNameNode(nameNode); + + if (!zkClient.exists(path)) { + zkClient.createPersistent(path, jsonObject.toJSONString()); + } + else { + zkClient.writeData(path, jsonObject.toJSONString()); + } + } + + private String getPathForNameNode(String nameNode) + { + return zkPath + "/" + nameNode; + } +} diff --git a/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperServersetMetastoreConfig.java b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperServersetMetastoreConfig.java new file mode 100644 index 0000000000000..b839a8d154042 --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/TestZookeeperServersetMetastoreConfig.java @@ -0,0 +1,67 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive; + +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Map; + +import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; + +public class TestZookeeperServersetMetastoreConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(ZookeeperServersetMetastoreConfig.class) + .setZookeeperMaxRetries(3) + .setZookeeperRetrySleepTime(500) + .setZookeeperMetastorePath(null) + .setZookeeperServerHostAndPort(null) + .setMaxTransport(128) + .setTransportIdleTimeout(300_000L) + .setTransportEvictInterval(10_000L) + .setTransportEvictNumTests(3)); + } + + @Test + public void testExplicitPropertyMappingsSingleMetastore() + { + Map properties = new ImmutableMap.Builder() + .put("hive.metastore.zookeeper.uri", "localhost:2181") + .put("hive.metastore.zookeeper.path", "/zookeeper/path/") + .put("hive.metastore.zookeeper.retry.sleeptime", "200") + .put("hive.metastore.zookeeper.max.retries", "2") + .put("hive.metastore.max-transport-num", "64") + .put("hive.metastore.transport-idle-timeout", "100000") + .put("hive.metastore.transport-eviction-interval", "1000") + .put("hive.metastore.transport-eviction-num-tests", "10") + .build(); + + ZookeeperServersetMetastoreConfig expected = new ZookeeperServersetMetastoreConfig() + .setZookeeperServerHostAndPort("localhost:2181") + .setZookeeperMetastorePath("/zookeeper/path/") + .setZookeeperRetrySleepTime(200) + .setZookeeperMaxRetries(2) + .setMaxTransport(64) + .setTransportIdleTimeout(100_000L) + .setTransportEvictInterval(1_000L) + .setTransportEvictNumTests(10); + + assertFullMapping(properties, expected); + } +} diff --git a/presto-hive/src/test/java/com/facebook/presto/twitter/hive/thrift/TestHiveThriftFieldIdResolver.java b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/thrift/TestHiveThriftFieldIdResolver.java new file mode 100644 index 0000000000000..6342a0c088e95 --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/thrift/TestHiveThriftFieldIdResolver.java @@ -0,0 +1,111 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive.thrift; + +import com.facebook.presto.spi.PrestoException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.Map; +import java.util.Properties; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertThrows; + +@Test +public class TestHiveThriftFieldIdResolver +{ + private static final Map STRUCT_FIELD_ID_AS_MAP = ImmutableMap.of( + "0", (short) 1, + "1", (short) 2, + "id", (short) 4); + + private static final Map LIST_FIELD_ID_AS_MAP = ImmutableMap.of( + "0", STRUCT_FIELD_ID_AS_MAP, + "id", (short) 5); + + private static final Map VERBOSE_PRIMARY_FIELD_ID_AS_MAP = ImmutableMap.of( + "id", (short) 6); + + private static final Map THRIFT_FIELD_ID_JSON_AS_MAP = ImmutableMap.builder() + .put("0", (short) 1) + .put("1", (short) 3) + .put("2", STRUCT_FIELD_ID_AS_MAP) + .put("3", LIST_FIELD_ID_AS_MAP) + .put("4", VERBOSE_PRIMARY_FIELD_ID_AS_MAP) + .build(); + + private final ObjectMapper objectMapper = new ObjectMapper(); + private final ThriftFieldIdResolverFactory resolverFactory = new HiveThriftFieldIdResolverFactory(); + + @Test + public void testDefaultResolver() + throws Exception + { + ThriftFieldIdResolver defaultResolver = resolverFactory.createResolver(new Properties()); + + for (int i = 0; i <= 5; ++i) { + assertEquals(defaultResolver.getThriftId(i), i + 1); + assertEquals(defaultResolver.getNestedResolver(i), defaultResolver); + } + for (int i = 5; i >= 0; --i) { + assertEquals(defaultResolver.getThriftId(i), i + 1); + assertEquals(defaultResolver.getNestedResolver(i), defaultResolver); + } + } + + @Test + public void testOptimizedResolver() + throws Exception + { + String json = objectMapper.writeValueAsString(THRIFT_FIELD_ID_JSON_AS_MAP); + Properties schema = new Properties(); + schema.setProperty(HiveThriftFieldIdResolverFactory.THRIFT_FIELD_ID_JSON, json); + ThriftFieldIdResolver resolver = resolverFactory.createResolver(schema); + + // primary field + assertEquals(resolver.getThriftId(0), THRIFT_FIELD_ID_JSON_AS_MAP.get("0")); + // discrete field + assertEquals(resolver.getThriftId(1), THRIFT_FIELD_ID_JSON_AS_MAP.get("1")); + + // nested field + ThriftFieldIdResolver nestedResolver = resolver.getNestedResolver(2); + Map field = (Map) THRIFT_FIELD_ID_JSON_AS_MAP.get("2"); + assertEquals(resolver.getThriftId(2), field.get("id")); + assertEquals(nestedResolver.getThriftId(0), field.get("0")); + assertEquals(nestedResolver.getThriftId(1), field.get("1")); + + // non-nested non-primary field + nestedResolver = resolver.getNestedResolver(3); + field = (Map) THRIFT_FIELD_ID_JSON_AS_MAP.get("3"); + assertEquals(resolver.getThriftId(3), field.get("id")); + + // non-primary nested field + nestedResolver = resolver.getNestedResolver(3); + field = (Map) THRIFT_FIELD_ID_JSON_AS_MAP.get("3"); + nestedResolver = nestedResolver.getNestedResolver(0); + field = (Map) field.get("0"); + assertEquals(nestedResolver.getThriftId(0), field.get("0")); + assertEquals(nestedResolver.getThriftId(1), field.get("1")); + + // verbose primary field + field = (Map) THRIFT_FIELD_ID_JSON_AS_MAP.get("4"); + assertEquals(resolver.getThriftId(4), field.get("id")); + + // non-existing field + assertThrows(PrestoException.class, () -> resolver.getThriftId(5)); + assertThrows(PrestoException.class, () -> resolver.getNestedResolver(5)); + } +} diff --git a/presto-hive/src/test/java/com/facebook/presto/twitter/hive/util/TestUtils.java b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/util/TestUtils.java new file mode 100644 index 0000000000000..379ad3877e325 --- /dev/null +++ b/presto-hive/src/test/java/com/facebook/presto/twitter/hive/util/TestUtils.java @@ -0,0 +1,30 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.hive.util; + +import java.io.IOException; +import java.net.ServerSocket; + +public final class TestUtils +{ + private TestUtils() {} + + public static int findUnusedPort() + throws IOException + { + try (ServerSocket socket = new ServerSocket(0)) { + return socket.getLocalPort(); + } + } +} diff --git a/presto-hive/src/test/resources/addressbook.thrift.lzo b/presto-hive/src/test/resources/addressbook.thrift.lzo new file mode 100644 index 0000000000000..bc982749d3ff9 Binary files /dev/null and b/presto-hive/src/test/resources/addressbook.thrift.lzo differ diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 8858fdeefaf91..75fecc836abf8 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-jdbc @@ -74,6 +74,12 @@ com.facebook.presto presto-main test + + + io.netty + netty + + diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 87d63c976f031..19cef1cb9ae1e 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index e971cf7561c1d..dd8c20437276a 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-kafka diff --git a/presto-kafka07/pom.xml b/presto-kafka07/pom.xml new file mode 100644 index 0000000000000..53363bd92c967 --- /dev/null +++ b/presto-kafka07/pom.xml @@ -0,0 +1,244 @@ + + + 4.0.0 + + + com.facebook.presto + presto-root + 0.208-tw-0.53 + + + presto-kafka07 + Presto - Kafka Connector for ver0.7 + presto-plugin + + + ${project.parent.basedir} + + + true + + + + + io.airlift + bootstrap + + + + io.airlift + json + + + + io.airlift + log + + + + io.airlift + configuration + + + + com.facebook.presto + presto-record-decoder + + + + com.google.guava + guava + + + + com.google.inject + guice + + + + javax.validation + validation-api + + + + com.twitter + rosette-kafka_2.11 + 0.7.2-21 + + + jsr305 + com.google.code.findbugs + + + zookeeper + org.apache.zookeeper + + + finagle-ostrich4_2.10 + com.twitter + + + commons-lang + commons-lang + + + + + + org.apache.zookeeper + zookeeper + + + + + joda-time + joda-time + + + + org.scala-lang + scala-library + 2.11.7 + + + + javax.annotation + javax.annotation-api + + + + javax.inject + javax.inject + + + + com.fasterxml.jackson.core + jackson-databind + + + + + com.facebook.presto + presto-spi + provided + + + + io.airlift + slice + provided + + + + io.airlift + units + provided + + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + + + io.airlift + log-manager + runtime + + + + + org.testng + testng + test + + + + io.airlift + testing + test + + + + com.facebook.presto + presto-main + test + + + + com.facebook.presto + presto-tpch + test + + + + com.facebook.presto + presto-client + test + + + + com.facebook.presto + presto-tests + test + + + + io.airlift.tpch + tpch + test + + + + com.github.sgroschupf + zkclient + 0.1 + + + log4j + log4j + + + + + + org.jetbrains + annotations + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + **/TestKafkaDistributed.java + + + + + + + + + ci + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + + + + + + diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaColumnHandle.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaColumnHandle.java new file mode 100644 index 0000000000000..b8ec023b24014 --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaColumnHandle.java @@ -0,0 +1,224 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.decoder.DecoderColumnHandle; +import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.type.Type; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +/** + * Kafka specific connector column handle. + */ +public final class KafkaColumnHandle + implements DecoderColumnHandle, Comparable +{ + private final String connectorId; + private final int ordinalPosition; + + /** + * Column Name + */ + private final String name; + + /** + * Column type + */ + private final Type type; + + /** + * Mapping hint for the decoder. Can be null. + */ + private final String mapping; + + /** + * Data format to use (selects the decoder). Can be null. + */ + private final String dataFormat; + + /** + * Additional format hint for the selected decoder. Selects a decoder subtype (e.g. which timestamp decoder). + */ + private final String formatHint; + + /** + * True if the key decoder should be used, false if the message decoder should be used. + */ + private final boolean keyDecoder; + + /** + * True if the column should be hidden. + */ + private final boolean hidden; + + /** + * True if the column is internal to the connector and not defined by a topic definition. + */ + private final boolean internal; + + @JsonCreator + public KafkaColumnHandle( + @JsonProperty("connectorId") String connectorId, + @JsonProperty("ordinalPosition") int ordinalPosition, + @JsonProperty("name") String name, + @JsonProperty("type") Type type, + @JsonProperty("mapping") String mapping, + @JsonProperty("dataFormat") String dataFormat, + @JsonProperty("formatHint") String formatHint, + @JsonProperty("keyDecoder") boolean keyDecoder, + @JsonProperty("hidden") boolean hidden, + @JsonProperty("internal") boolean internal) + + { + this.connectorId = requireNonNull(connectorId, "connectorId is null"); + this.ordinalPosition = ordinalPosition; + this.name = requireNonNull(name, "name is null"); + this.type = requireNonNull(type, "type is null"); + this.mapping = mapping; + this.dataFormat = dataFormat; + this.formatHint = formatHint; + this.keyDecoder = keyDecoder; + this.hidden = hidden; + this.internal = internal; + } + + @JsonProperty + public String getConnectorId() + { + return connectorId; + } + + @JsonProperty + public int getOrdinalPosition() + { + return ordinalPosition; + } + + @Override + @JsonProperty + public String getName() + { + return name; + } + + @Override + @JsonProperty + public Type getType() + { + return type; + } + + @Override + @JsonProperty + public String getMapping() + { + return mapping; + } + + @Override + @JsonProperty + public String getDataFormat() + { + return dataFormat; + } + + @Override + @JsonProperty + public String getFormatHint() + { + return formatHint; + } + + @JsonProperty + public boolean isKeyDecoder() + { + return keyDecoder; + } + + @JsonProperty + public boolean isHidden() + { + return hidden; + } + + @Override + @JsonProperty + public boolean isInternal() + { + return internal; + } + + ColumnMetadata getColumnMetadata() + { + return new ColumnMetadata(name, type, null, hidden); + } + + @Override + public int hashCode() + { + return Objects.hash(connectorId, ordinalPosition, name, type, mapping, dataFormat, formatHint, keyDecoder, hidden, internal); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + KafkaColumnHandle other = (KafkaColumnHandle) obj; + return Objects.equals(this.connectorId, other.connectorId) && + Objects.equals(this.ordinalPosition, other.ordinalPosition) && + Objects.equals(this.name, other.name) && + Objects.equals(this.type, other.type) && + Objects.equals(this.mapping, other.mapping) && + Objects.equals(this.dataFormat, other.dataFormat) && + Objects.equals(this.formatHint, other.formatHint) && + Objects.equals(this.keyDecoder, other.keyDecoder) && + Objects.equals(this.hidden, other.hidden) && + Objects.equals(this.internal, other.internal); + } + + @Override + public int compareTo(KafkaColumnHandle otherHandle) + { + return Integer.compare(this.getOrdinalPosition(), otherHandle.getOrdinalPosition()); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("connectorId", connectorId) + .add("ordinalPosition", ordinalPosition) + .add("name", name) + .add("type", type) + .add("mapping", mapping) + .add("dataFormat", dataFormat) + .add("formatHint", formatHint) + .add("keyDecoder", keyDecoder) + .add("hidden", hidden) + .add("internal", internal) + .toString(); + } +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaConnector.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaConnector.java new file mode 100644 index 0000000000000..69cfe62e87379 --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaConnector.java @@ -0,0 +1,92 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.spi.connector.Connector; +import com.facebook.presto.spi.connector.ConnectorMetadata; +import com.facebook.presto.spi.connector.ConnectorRecordSetProvider; +import com.facebook.presto.spi.connector.ConnectorSplitManager; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.facebook.presto.spi.transaction.IsolationLevel; +import io.airlift.bootstrap.LifeCycleManager; +import io.airlift.log.Logger; + +import javax.inject.Inject; + +import static com.facebook.presto.spi.transaction.IsolationLevel.READ_COMMITTED; +import static com.facebook.presto.spi.transaction.IsolationLevel.checkConnectorSupports; +import static java.util.Objects.requireNonNull; + +/** + * Kafka specific implementation of the Presto Connector SPI. This is a read only connector. + */ +public class KafkaConnector + implements Connector +{ + private static final Logger log = Logger.get(KafkaConnector.class); + + private final LifeCycleManager lifeCycleManager; + private final KafkaMetadata metadata; + private final KafkaSplitManager splitManager; + private final KafkaRecordSetProvider recordSetProvider; + + @Inject + public KafkaConnector( + LifeCycleManager lifeCycleManager, + KafkaMetadata metadata, + KafkaSplitManager splitManager, + KafkaRecordSetProvider recordSetProvider) + { + this.lifeCycleManager = requireNonNull(lifeCycleManager, "lifeCycleManager is null"); + this.metadata = requireNonNull(metadata, "metadata is null"); + this.splitManager = requireNonNull(splitManager, "splitManager is null"); + this.recordSetProvider = requireNonNull(recordSetProvider, "recordSetProvider is null"); + } + + @Override + public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly) + { + checkConnectorSupports(READ_COMMITTED, isolationLevel); + return KafkaTransactionHandle.INSTANCE; + } + + @Override + public ConnectorMetadata getMetadata(ConnectorTransactionHandle transactionHandle) + { + return metadata; + } + + @Override + public ConnectorSplitManager getSplitManager() + { + return splitManager; + } + + @Override + public ConnectorRecordSetProvider getRecordSetProvider() + { + return recordSetProvider; + } + + @Override + public final void shutdown() + { + try { + lifeCycleManager.stop(); + } + catch (Exception e) { + log.error(e, "Error shutting down connector"); + } + } +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaConnectorConfig.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaConnectorConfig.java new file mode 100644 index 0000000000000..de5615de21676 --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaConnectorConfig.java @@ -0,0 +1,204 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.spi.HostAddress; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableSet; +import io.airlift.configuration.Config; +import io.airlift.units.DataSize; +import io.airlift.units.DataSize.Unit; +import io.airlift.units.Duration; +import io.airlift.units.MinDuration; + +import javax.validation.constraints.NotNull; + +import java.io.File; +import java.util.Set; + +import static com.google.common.collect.Iterables.transform; + +public class KafkaConnectorConfig +{ + private static final int KAFKA_DEFAULT_PORT = 9092; + + /** + * Seed nodes for Kafka cluster. At least one must exist. + */ + private Set nodes = ImmutableSet.of(); + + /** + * Timeout to connect to Kafka. + */ + private Duration kafkaConnectTimeout = Duration.valueOf("10s"); + + /** + * Buffer size for connecting to Kafka. + */ + private DataSize kafkaBufferSize = new DataSize(64, Unit.KILOBYTE); + + /** + * The schema name to use in the connector. + */ + private String defaultSchema = "default"; + + /** + * Set of tables known to this connector. For each table, a description file may be present in the catalog folder which describes columns for the given topic. + */ + private Set tableNames = ImmutableSet.of(); + + /** + * Folder holding the JSON description files for Kafka topics. + */ + private File tableDescriptionDir = new File("etc/kafka07/"); + + /** + * Whether internal columns are shown in table metadata or not. Default is no. + */ + private boolean hideInternalColumns = true; + + /** + * ZK endpoint for getting broker list + */ + private String zkEndpoint = ""; + + /** + * Fetch size + */ + private int fetchSize = 10 * 1024 * 1024; + + @NotNull + public File getTableDescriptionDir() + { + return tableDescriptionDir; + } + + @Config("kafka.table-description-dir") + public KafkaConnectorConfig setTableDescriptionDir(File tableDescriptionDir) + { + this.tableDescriptionDir = tableDescriptionDir; + return this; + } + + @NotNull + public Set getTableNames() + { + return tableNames; + } + + @Config("kafka.table-names") + public KafkaConnectorConfig setTableNames(String tableNames) + { + this.tableNames = ImmutableSet.copyOf(Splitter.on(',').omitEmptyStrings().trimResults().split(tableNames)); + return this; + } + + @NotNull + public String getDefaultSchema() + { + return defaultSchema; + } + + @Config("kafka.default-schema") + public KafkaConnectorConfig setDefaultSchema(String defaultSchema) + { + this.defaultSchema = defaultSchema; + return this; + } + + public Set getNodes() + { + return nodes; + } + + @Config("kafka.nodes") + public KafkaConnectorConfig setNodes(String nodes) + { + this.nodes = (nodes == null) ? null : parseNodes(nodes); + return this; + } + + @MinDuration("1s") + public Duration getKafkaConnectTimeout() + { + return kafkaConnectTimeout; + } + + @Config("kafka.connect-timeout") + public KafkaConnectorConfig setKafkaConnectTimeout(String kafkaConnectTimeout) + { + this.kafkaConnectTimeout = Duration.valueOf(kafkaConnectTimeout); + return this; + } + + public DataSize getKafkaBufferSize() + { + return kafkaBufferSize; + } + + @Config("kafka.buffer-size") + public KafkaConnectorConfig setKafkaBufferSize(String kafkaBufferSize) + { + this.kafkaBufferSize = DataSize.valueOf(kafkaBufferSize); + return this; + } + + public boolean isHideInternalColumns() + { + return hideInternalColumns; + } + + @Config("kafka.hide-internal-columns") + public KafkaConnectorConfig setHideInternalColumns(boolean hideInternalColumns) + { + this.hideInternalColumns = hideInternalColumns; + return this; + } + + @NotNull + public String getZkEndpoint() + { + return zkEndpoint; + } + + @Config("kafka.zk-endpoint") + public KafkaConnectorConfig setZkEndpoint(String zkEndpoint) + { + this.zkEndpoint = zkEndpoint; + return this; + } + + public int getFetchSize() + { + return fetchSize; + } + + @Config("kafka.fetch-size") + public KafkaConnectorConfig setFetchSize(int fetchSize) + { + this.fetchSize = fetchSize; + return this; + } + + public static ImmutableSet parseNodes(String nodes) + { + Splitter splitter = Splitter.on(',').omitEmptyStrings().trimResults(); + return ImmutableSet.copyOf(transform(splitter.split(nodes), KafkaConnectorConfig::toHostAddress)); + } + + private static HostAddress toHostAddress(String value) + { + return HostAddress.fromString(value).withDefaultPort(KAFKA_DEFAULT_PORT); + } +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaConnectorFactory.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaConnectorFactory.java new file mode 100644 index 0000000000000..b9721f0459f83 --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaConnectorFactory.java @@ -0,0 +1,95 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.spi.ConnectorHandleResolver; +import com.facebook.presto.spi.NodeManager; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.connector.Connector; +import com.facebook.presto.spi.connector.ConnectorContext; +import com.facebook.presto.spi.connector.ConnectorFactory; +import com.facebook.presto.spi.type.TypeManager; +import com.google.common.base.Throwables; +import com.google.inject.Injector; +import com.google.inject.Scopes; +import com.google.inject.TypeLiteral; +import io.airlift.bootstrap.Bootstrap; +import io.airlift.json.JsonModule; + +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; + +import static java.util.Objects.requireNonNull; + +/** + * Creates Kafka Connectors based off connectorId and specific configuration. + */ +public class KafkaConnectorFactory + implements ConnectorFactory +{ + private final Optional>> tableDescriptionSupplier; + + KafkaConnectorFactory(Optional>> tableDescriptionSupplier) + { + this.tableDescriptionSupplier = requireNonNull(tableDescriptionSupplier, "tableDescriptionSupplier is null"); + } + + @Override + public String getName() + { + return "kafka07"; + } + + @Override + public ConnectorHandleResolver getHandleResolver() + { + return new KafkaHandleResolver(); + } + + @Override + public Connector create(String connectorId, Map config, ConnectorContext context) + { + requireNonNull(connectorId, "connectorId is null"); + requireNonNull(config, "config is null"); + + try { + Bootstrap app = new Bootstrap( + new JsonModule(), + new KafkaConnectorModule(), + binder -> { + binder.bind(KafkaConnectorId.class).toInstance(new KafkaConnectorId(connectorId)); + binder.bind(TypeManager.class).toInstance(context.getTypeManager()); + binder.bind(NodeManager.class).toInstance(context.getNodeManager()); + + if (tableDescriptionSupplier.isPresent()) { + binder.bind(new TypeLiteral>>() {}).toInstance(tableDescriptionSupplier.get()); + } + else { + binder.bind(new TypeLiteral>>() {}).to(KafkaTableDescriptionSupplier.class).in(Scopes.SINGLETON); + } + }); + + Injector injector = app.strictConfig() + .doNotInitializeLogging() + .setRequiredConfigurationProperties(config) + .initialize(); + + return injector.getInstance(KafkaConnector.class); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaConnectorId.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaConnectorId.java new file mode 100644 index 0000000000000..3470980df0735 --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaConnectorId.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public class KafkaConnectorId +{ + private final String connectorId; + + public KafkaConnectorId(String connectorId) + { + this.connectorId = requireNonNull(connectorId, "connectorId is null"); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + KafkaConnectorId other = (KafkaConnectorId) obj; + return Objects.equals(this.connectorId, other.connectorId); + } + + @Override + public int hashCode() + { + return Objects.hash(connectorId); + } + + @Override + public String toString() + { + return connectorId; + } +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaConnectorModule.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaConnectorModule.java new file mode 100644 index 0000000000000..8afa5068eb82e --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaConnectorModule.java @@ -0,0 +1,78 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.decoder.DecoderModule; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Scopes; + +import javax.inject.Inject; + +import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; +import static io.airlift.configuration.ConfigBinder.configBinder; +import static io.airlift.json.JsonBinder.jsonBinder; +import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; +import static java.util.Objects.requireNonNull; + +/** + * Guice module for the Apache Kafka connector. + */ +public class KafkaConnectorModule + implements Module +{ + @Override + public void configure(Binder binder) + { + binder.bind(KafkaConnector.class).in(Scopes.SINGLETON); + + binder.bind(KafkaMetadata.class).in(Scopes.SINGLETON); + binder.bind(KafkaSplitManager.class).in(Scopes.SINGLETON); + binder.bind(KafkaRecordSetProvider.class).in(Scopes.SINGLETON); + + binder.bind(KafkaSimpleConsumerManager.class).in(Scopes.SINGLETON); + + configBinder(binder).bindConfig(KafkaConnectorConfig.class); + + jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class); + jsonCodecBinder(binder).bindJsonCodec(KafkaTopicDescription.class); + + binder.install(new DecoderModule()); + } + + public static final class TypeDeserializer + extends FromStringDeserializer + { + private static final long serialVersionUID = 1L; + + private final TypeManager typeManager; + + @Inject + public TypeDeserializer(TypeManager typeManager) + { + super(Type.class); + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + } + + @Override + protected Type _deserialize(String value, DeserializationContext context) + { + return typeManager.getType(parseTypeSignature(value)); + } + } +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaErrorCode.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaErrorCode.java new file mode 100644 index 0000000000000..9338b98492116 --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaErrorCode.java @@ -0,0 +1,42 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.spi.ErrorCode; +import com.facebook.presto.spi.ErrorCodeSupplier; +import com.facebook.presto.spi.ErrorType; + +import static com.facebook.presto.spi.ErrorType.EXTERNAL; + +/** + * Kafka connector specific error codes. + */ +public enum KafkaErrorCode + implements ErrorCodeSupplier +{ + KAFKA_SPLIT_ERROR(0, EXTERNAL); + + private final ErrorCode errorCode; + + KafkaErrorCode(int code, ErrorType type) + { + errorCode = new ErrorCode(code + 0x0102_0000, name(), type); + } + + @Override + public ErrorCode toErrorCode() + { + return errorCode; + } +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaHandleResolver.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaHandleResolver.java new file mode 100644 index 0000000000000..539b082701298 --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaHandleResolver.java @@ -0,0 +1,89 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorHandleResolver; +import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.ConnectorTableHandle; +import com.facebook.presto.spi.ConnectorTableLayoutHandle; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +/** + * Kafka specific {@link com.facebook.presto.spi.ConnectorHandleResolver} implementation. + */ +public class KafkaHandleResolver + implements ConnectorHandleResolver +{ + @Override + public Class getTableHandleClass() + { + return KafkaTableHandle.class; + } + + @Override + public Class getColumnHandleClass() + { + return KafkaColumnHandle.class; + } + + @Override + public Class getSplitClass() + { + return KafkaSplit.class; + } + + @Override + public Class getTableLayoutHandleClass() + { + return KafkaTableLayoutHandle.class; + } + + @Override + public Class getTransactionHandleClass() + { + return KafkaTransactionHandle.class; + } + + static KafkaTableHandle convertTableHandle(ConnectorTableHandle tableHandle) + { + requireNonNull(tableHandle, "tableHandle is null"); + checkArgument(tableHandle instanceof KafkaTableHandle, "tableHandle is not an instance of KafkaTableHandle"); + return (KafkaTableHandle) tableHandle; + } + + static KafkaColumnHandle convertColumnHandle(ColumnHandle columnHandle) + { + requireNonNull(columnHandle, "columnHandle is null"); + checkArgument(columnHandle instanceof KafkaColumnHandle, "columnHandle is not an instance of KafkaColumnHandle"); + return (KafkaColumnHandle) columnHandle; + } + + static KafkaSplit convertSplit(ConnectorSplit split) + { + requireNonNull(split, "split is null"); + checkArgument(split instanceof KafkaSplit, "split is not an instance of KafkaSplit"); + return (KafkaSplit) split; + } + + static KafkaTableLayoutHandle convertLayout(ConnectorTableLayoutHandle layout) + { + requireNonNull(layout, "layout is null"); + checkArgument(layout instanceof KafkaTableLayoutHandle, "layout is not an instance of KafkaTableLayoutHandle"); + return (KafkaTableLayoutHandle) layout; + } +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaInternalFieldDescription.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaInternalFieldDescription.java new file mode 100644 index 0000000000000..fb74806e00274 --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaInternalFieldDescription.java @@ -0,0 +1,153 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.type.BigintType; +import com.facebook.presto.spi.type.BooleanType; +import com.facebook.presto.spi.type.Type; + +import java.util.Map; + +import static com.facebook.presto.spi.type.VarcharType.createUnboundedVarcharType; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Strings.isNullOrEmpty; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static java.util.Arrays.stream; +import static java.util.Objects.requireNonNull; +import static java.util.function.Function.identity; + +/** + * Describes an internal (managed by the connector) field which is added to each table row. The definition itself makes the row + * show up in the tables (the columns are hidden by default, so they must be explicitly selected) but unless the field is hooked in using the + * forBooleanValue/forLongValue/forBytesValue methods and the resulting FieldValueProvider is then passed into the appropriate row decoder, the fields + * will be null. Most values are assigned in the {@link com.facebook.presto.kafka.KafkaRecordSet}. + */ +public enum KafkaInternalFieldDescription +{ + /** + * _partition_id - Kafka partition id. + */ + PARTITION_ID_FIELD("_partition_id", BigintType.BIGINT, "Partition Id"), + + /** + * _partition_offset - The current offset of the message in the partition. + */ + PARTITION_OFFSET_FIELD("_partition_offset", BigintType.BIGINT, "Offset for the message within the partition"), + + /** + * _segment_start - Kafka start offset for the segment which contains the current message. This is per-partition. + */ + SEGMENT_START_FIELD("_segment_start", BigintType.BIGINT, "Segment start offset"), + + /** + * _segment_end - Kafka end offset for the segment which contains the current message. This is per-partition. The end offset is the first offset that is *not* in the segment. + */ + SEGMENT_END_FIELD("_segment_end", BigintType.BIGINT, "Segment end offset"), + + /** + * _segment_count - Running count of messages in a segment. + */ + SEGMENT_COUNT_FIELD("_segment_count", BigintType.BIGINT, "Running message count per segment"), + + /** + * _message_corrupt - True if the row converter could not read the a message. May be null if the row converter does not set a value (e.g. the dummy row converter does not). + */ + MESSAGE_CORRUPT_FIELD("_message_corrupt", BooleanType.BOOLEAN, "Message data is corrupt"), + + /** + * _message - Represents the full topic as a text column. Format is UTF-8 which may be wrong for some topics. TODO: make charset configurable. + */ + MESSAGE_FIELD("_message", createUnboundedVarcharType(), "Message text"), + + /** + * _message_length - length in bytes of the message. + */ + MESSAGE_LENGTH_FIELD("_message_length", BigintType.BIGINT, "Total number of message bytes"), + + /** + * _key_corrupt - True if the row converter could not read the a key. May be null if the row converter does not set a value (e.g. the dummy row converter does not). + */ + KEY_CORRUPT_FIELD("_key_corrupt", BooleanType.BOOLEAN, "Key data is corrupt"), + + /** + * _key - Represents the key as a text column. Format is UTF-8 which may be wrong for topics. TODO: make charset configurable. + */ + KEY_FIELD("_key", createUnboundedVarcharType(), "Key text"), + + /** + * _key_length - length in bytes of the key. + */ + KEY_LENGTH_FIELD("_key_length", BigintType.BIGINT, "Total number of key bytes"), + + /** + * _timestamp - offset timestamp, used to narrow scan range + */ + OFFSET_TIMESTAMP_FIELD("_timestamp", BigintType.BIGINT, "Offset Timestamp"); + + private static final Map BY_COLUMN_NAME = + stream(KafkaInternalFieldDescription.values()) + .collect(toImmutableMap(KafkaInternalFieldDescription::getColumnName, identity())); + + public static KafkaInternalFieldDescription forColumnName(String columnName) + { + KafkaInternalFieldDescription description = BY_COLUMN_NAME.get(columnName); + checkArgument(description != null, "Unknown internal column name %s", columnName); + return description; + } + + private final String columnName; + private final Type type; + private final String comment; + + KafkaInternalFieldDescription( + String columnName, + Type type, + String comment) + { + checkArgument(!isNullOrEmpty(columnName), "name is null or is empty"); + this.columnName = columnName; + this.type = requireNonNull(type, "type is null"); + this.comment = requireNonNull(comment, "comment is null"); + } + + public String getColumnName() + { + return columnName; + } + + public Type getType() + { + return type; + } + + KafkaColumnHandle getColumnHandle(String connectorId, int index, boolean hidden) + { + return new KafkaColumnHandle(connectorId, + index, + getColumnName(), + getType(), + null, + null, + null, + false, + hidden, + true); + } + + ColumnMetadata getColumnMetadata(boolean hidden) + { + return new ColumnMetadata(columnName, type, comment, hidden); + } +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaMetadata.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaMetadata.java new file mode 100644 index 0000000000000..225c619c28aee --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaMetadata.java @@ -0,0 +1,274 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.decoder.dummy.DummyRowDecoder; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.ConnectorTableHandle; +import com.facebook.presto.spi.ConnectorTableLayout; +import com.facebook.presto.spi.ConnectorTableLayoutHandle; +import com.facebook.presto.spi.ConnectorTableLayoutResult; +import com.facebook.presto.spi.ConnectorTableMetadata; +import com.facebook.presto.spi.Constraint; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.SchemaTablePrefix; +import com.facebook.presto.spi.TableNotFoundException; +import com.facebook.presto.spi.connector.ConnectorMetadata; +import com.facebook.presto.spi.predicate.Domain; +import com.facebook.presto.spi.predicate.Marker; +import com.facebook.presto.spi.predicate.Range; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import io.airlift.log.Logger; + +import javax.inject.Inject; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Supplier; + +import static com.facebook.presto.kafka.KafkaHandleResolver.convertColumnHandle; +import static com.facebook.presto.kafka.KafkaHandleResolver.convertTableHandle; +import static java.util.Objects.requireNonNull; + +/** + * Manages the Kafka connector specific metadata information. The Connector provides an additional set of columns + * for each table that are created as hidden columns. See {@link KafkaInternalFieldDescription} for a list + * of per-topic additional columns. + */ +public class KafkaMetadata + implements ConnectorMetadata +{ + private static final Logger log = Logger.get(KafkaMetadata.class); + + private final String connectorId; + private final boolean hideInternalColumns; + private final Map tableDescriptions; + private final Set internalFieldDescriptions; + + @Inject + public KafkaMetadata( + KafkaConnectorId connectorId, + KafkaConnectorConfig kafkaConnectorConfig, + Supplier> kafkaTableDescriptionSupplier, + Set internalFieldDescriptions) + { + this.connectorId = requireNonNull(connectorId, "connectorId is null").toString(); + + requireNonNull(kafkaConnectorConfig, "kafkaConfig is null"); + this.hideInternalColumns = kafkaConnectorConfig.isHideInternalColumns(); + + requireNonNull(kafkaTableDescriptionSupplier, "kafkaTableDescriptionSupplier is null"); + this.tableDescriptions = kafkaTableDescriptionSupplier.get(); + this.internalFieldDescriptions = requireNonNull(internalFieldDescriptions, "internalFieldDescriptions is null"); + } + + @Override + public List listSchemaNames(ConnectorSession session) + { + ImmutableSet.Builder builder = ImmutableSet.builder(); + for (SchemaTableName tableName : tableDescriptions.keySet()) { + builder.add(tableName.getSchemaName()); + } + return ImmutableList.copyOf(builder.build()); + } + + @Override + public KafkaTableHandle getTableHandle(ConnectorSession session, SchemaTableName schemaTableName) + { + KafkaTopicDescription table = tableDescriptions.get(schemaTableName); + if (table == null) { + return null; + } + + return new KafkaTableHandle(connectorId, + schemaTableName.getSchemaName(), + schemaTableName.getTableName(), + table.getTopicName(), + getDataFormat(table.getKey()), + getDataFormat(table.getMessage())); + } + + private static String getDataFormat(KafkaTopicFieldGroup fieldGroup) + { + return (fieldGroup == null) ? DummyRowDecoder.NAME : fieldGroup.getDataFormat(); + } + + @Override + public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle tableHandle) + { + return getTableMetadata(convertTableHandle(tableHandle).toSchemaTableName()); + } + + @Override + public List listTables(ConnectorSession session, String schemaNameOrNull) + { + ImmutableList.Builder builder = ImmutableList.builder(); + for (SchemaTableName tableName : tableDescriptions.keySet()) { + if (schemaNameOrNull == null || tableName.getSchemaName().equals(schemaNameOrNull)) { + builder.add(tableName); + } + } + + return builder.build(); + } + + @SuppressWarnings("ValueOfIncrementOrDecrementUsed") + @Override + public Map getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle) + { + KafkaTableHandle kafkaTableHandle = convertTableHandle(tableHandle); + + KafkaTopicDescription kafkaTopicDescription = tableDescriptions.get(kafkaTableHandle.toSchemaTableName()); + if (kafkaTopicDescription == null) { + throw new TableNotFoundException(kafkaTableHandle.toSchemaTableName()); + } + + ImmutableMap.Builder columnHandles = ImmutableMap.builder(); + + int index = 0; + KafkaTopicFieldGroup key = kafkaTopicDescription.getKey(); + if (key != null) { + List fields = key.getFields(); + if (fields != null) { + for (KafkaTopicFieldDescription kafkaTopicFieldDescription : fields) { + columnHandles.put(kafkaTopicFieldDescription.getName(), kafkaTopicFieldDescription.getColumnHandle(connectorId, true, index++)); + } + } + } + + KafkaTopicFieldGroup message = kafkaTopicDescription.getMessage(); + if (message != null) { + List fields = message.getFields(); + if (fields != null) { + for (KafkaTopicFieldDescription kafkaTopicFieldDescription : fields) { + columnHandles.put(kafkaTopicFieldDescription.getName(), kafkaTopicFieldDescription.getColumnHandle(connectorId, false, index++)); + } + } + } + + for (KafkaInternalFieldDescription kafkaInternalFieldDescription : internalFieldDescriptions) { + columnHandles.put(kafkaInternalFieldDescription.getColumnName(), kafkaInternalFieldDescription.getColumnHandle(connectorId, index++, hideInternalColumns)); + } + + return columnHandles.build(); + } + + @Override + public Map> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix) + { + requireNonNull(prefix, "prefix is null"); + + ImmutableMap.Builder> columns = ImmutableMap.builder(); + + List tableNames = prefix.getSchemaName() == null ? listTables(session, Optional.empty()) : ImmutableList.of(new SchemaTableName(prefix.getSchemaName(), prefix.getTableName())); + + for (SchemaTableName tableName : tableNames) { + ConnectorTableMetadata tableMetadata = getTableMetadata(tableName); + // table can disappear during listing operation + if (tableMetadata != null) { + columns.put(tableName, tableMetadata.getColumns()); + } + } + return columns.build(); + } + + @Override + public ColumnMetadata getColumnMetadata(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle columnHandle) + { + convertTableHandle(tableHandle); + return convertColumnHandle(columnHandle).getColumnMetadata(); + } + + @Override + public List getTableLayouts(ConnectorSession session, ConnectorTableHandle table, Constraint constraint, Optional> desiredColumns) + { + KafkaTableHandle handle = convertTableHandle(table); + + Long startTs = null; + Long endTs = null; + Optional> domains = constraint.getSummary().getDomains(); + log.info(constraint.getSummary().toString(null)); + if (domains.isPresent()) { + Map columnHandleDomainMap = domains.get(); + for (Map.Entry entry : columnHandleDomainMap.entrySet()) { + if (entry.getKey() instanceof KafkaColumnHandle && ((KafkaColumnHandle) entry.getKey()).getName().equals(KafkaInternalFieldDescription.OFFSET_TIMESTAMP_FIELD.getColumnName())) { + Range span = entry.getValue().getValues().getRanges().getSpan(); + Marker low = span.getLow(); + Marker high = span.getHigh(); + if (!low.isLowerUnbounded()) { + startTs = (Long) low.getValue(); + } + if (!high.isUpperUnbounded()) { + endTs = (Long) high.getValue(); + } + } + + log.info("K: %s\tV: %s", entry.getKey().toString(), entry.getValue().toString()); + } + } + + log.info("startTs: %s, endTs: %s", startTs, endTs); + ConnectorTableLayout layout = new ConnectorTableLayout(new KafkaTableLayoutHandle(handle, startTs, endTs)); + return ImmutableList.of(new ConnectorTableLayoutResult(layout, constraint.getSummary())); + } + + @Override + public ConnectorTableLayout getTableLayout(ConnectorSession session, ConnectorTableLayoutHandle handle) + { + return new ConnectorTableLayout(handle); + } + + @SuppressWarnings("ValueOfIncrementOrDecrementUsed") + private ConnectorTableMetadata getTableMetadata(SchemaTableName schemaTableName) + { + KafkaTopicDescription table = tableDescriptions.get(schemaTableName); + if (table == null) { + throw new TableNotFoundException(schemaTableName); + } + + ImmutableList.Builder builder = ImmutableList.builder(); + + KafkaTopicFieldGroup key = table.getKey(); + if (key != null) { + List fields = key.getFields(); + if (fields != null) { + for (KafkaTopicFieldDescription fieldDescription : fields) { + builder.add(fieldDescription.getColumnMetadata()); + } + } + } + + KafkaTopicFieldGroup message = table.getMessage(); + if (message != null) { + List fields = message.getFields(); + if (fields != null) { + for (KafkaTopicFieldDescription fieldDescription : fields) { + builder.add(fieldDescription.getColumnMetadata()); + } + } + } + + for (KafkaInternalFieldDescription fieldDescription : internalFieldDescriptions) { + builder.add(fieldDescription.getColumnMetadata(hideInternalColumns)); + } + + return new ConnectorTableMetadata(schemaTableName, builder.build()); + } +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaPlugin.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaPlugin.java new file mode 100644 index 0000000000000..284424a9a7fa4 --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaPlugin.java @@ -0,0 +1,47 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.spi.Plugin; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.connector.ConnectorFactory; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; + +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; + +import static java.util.Objects.requireNonNull; + +/** + * Presto plugin to use Apache Kafka as a data source. + */ +public class KafkaPlugin + implements Plugin +{ + private Optional>> tableDescriptionSupplier = Optional.empty(); + + @VisibleForTesting + public synchronized void setTableDescriptionSupplier(Supplier> tableDescriptionSupplier) + { + this.tableDescriptionSupplier = Optional.of(requireNonNull(tableDescriptionSupplier, "tableDescriptionSupplier is null")); + } + + @Override + public synchronized Iterable getConnectorFactories() + { + return ImmutableList.of(new KafkaConnectorFactory(tableDescriptionSupplier)); + } +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaRecordSet.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaRecordSet.java new file mode 100644 index 0000000000000..2827d8bd56095 --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaRecordSet.java @@ -0,0 +1,353 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.decoder.DecoderColumnHandle; +import com.facebook.presto.decoder.FieldValueProvider; +import com.facebook.presto.decoder.RowDecoder; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.RecordCursor; +import com.facebook.presto.spi.RecordSet; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.type.Type; +import com.google.common.collect.ImmutableList; +import io.airlift.log.Logger; +import io.airlift.slice.Slice; +import kafka.api.FetchRequest; +import kafka.api.OffsetRequest; +import kafka.common.ErrorMapping; +import kafka.common.OffsetOutOfRangeException; +import kafka.javaapi.consumer.SimpleConsumer; +import kafka.javaapi.message.ByteBufferMessageSet; +import kafka.message.MessageAndOffset; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; + +import static com.facebook.presto.decoder.FieldValueProviders.booleanValueProvider; +import static com.facebook.presto.decoder.FieldValueProviders.bytesValueProvider; +import static com.facebook.presto.decoder.FieldValueProviders.longValueProvider; +import static com.facebook.presto.kafka.KafkaErrorCode.KAFKA_SPLIT_ERROR; +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +/** + * Kafka specific record set. Returns a cursor for a topic which iterates over a Kafka partition segment. + */ +public class KafkaRecordSet + implements RecordSet +{ + private static final Logger log = Logger.get(KafkaRecordSet.class); + + private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + + private final KafkaSplit split; + private final KafkaSimpleConsumerManager consumerManager; + private final int fetchSize; + + private final RowDecoder keyDecoder; + private final RowDecoder messageDecoder; + + private final List columnHandles; + private final List columnTypes; + + KafkaRecordSet(KafkaSplit split, + KafkaSimpleConsumerManager consumerManager, + List columnHandles, + RowDecoder keyDecoder, + RowDecoder messageDecoder, + int fetchSize) + { + this.split = requireNonNull(split, "split is null"); + + this.consumerManager = requireNonNull(consumerManager, "consumerManager is null"); + + this.keyDecoder = requireNonNull(keyDecoder, "rowDecoder is null"); + this.messageDecoder = requireNonNull(messageDecoder, "rowDecoder is null"); + + this.columnHandles = requireNonNull(columnHandles, "columnHandles is null"); + + ImmutableList.Builder typeBuilder = ImmutableList.builder(); + + for (DecoderColumnHandle handle : columnHandles) { + typeBuilder.add(handle.getType()); + } + + this.columnTypes = typeBuilder.build(); + this.fetchSize = fetchSize; + } + + @Override + public List getColumnTypes() + { + return columnTypes; + } + + @Override + public RecordCursor cursor() + { + return new KafkaRecordCursor(split.getStartTs(), split.getEndTs()); + } + + public class KafkaRecordCursor + implements RecordCursor + { + private long totalBytes; + private long totalMessages; + private long cursorOffset = split.getStart(); + private Iterator messageAndOffsetIterator; + private long fetchedSize; + private final AtomicBoolean reported = new AtomicBoolean(); + + private final long startTs; + private final long endTs; + + private FieldValueProvider[] currentRowValues = new FieldValueProvider[columnHandles.size()]; + + KafkaRecordCursor(long startTs, long endTs) + { + this.startTs = startTs; + this.endTs = endTs; + } + + @Override + public long getCompletedBytes() + { + return totalBytes; + } + + @Override + public long getReadTimeNanos() + { + return 0; + } + + @Override + public Type getType(int field) + { + checkArgument(field < columnHandles.size(), "Invalid field index"); + return columnHandles.get(field).getType(); + } + + @Override + public boolean advanceNextPosition() + { + while (true) { + if (cursorOffset >= split.getEnd()) { + return endOfData(1); // Split end is exclusive. + } + + try { + // Create a fetch request + openFetchRequest(); + if (cursorOffset >= split.getEnd()) { + return endOfData(2); // Split end is exclusive. + } + if (messageAndOffsetIterator.hasNext()) { + MessageAndOffset currentMessageAndOffset = messageAndOffsetIterator.next(); + return nextRow(currentMessageAndOffset); + } + } + catch (OffsetOutOfRangeException e) { + e.printStackTrace(); + return endOfData(4); + } + messageAndOffsetIterator = null; + } + } + + private boolean endOfData(int from) + { + if (!reported.getAndSet(true)) { + log.info("Found (from %d) a total of %d messages with %d bytes (%d compressed bytes expected). Last Offset: %d (%d, %d)", + from, totalMessages, totalBytes, split.getEnd() - split.getStart(), + cursorOffset, split.getStart(), split.getEnd()); + } + return false; + } + + private boolean nextRow(MessageAndOffset messageAndOffset) + { + totalBytes += messageAndOffset.message().payloadSize(); + totalMessages++; + + byte[] keyData = EMPTY_BYTE_ARRAY; + byte[] messageData = EMPTY_BYTE_ARRAY; + + ByteBuffer message = messageAndOffset.message().payload(); + if (message != null) { + messageData = new byte[message.remaining()]; + message.get(messageData); + } + + Map currentRowValuesMap = new HashMap<>(); + + Optional> decodedKey = keyDecoder.decodeRow(keyData, null); + Optional> decodedValue = messageDecoder.decodeRow(messageData, null); + + for (DecoderColumnHandle columnHandle : columnHandles) { + if (columnHandle.isInternal()) { + KafkaInternalFieldDescription fieldDescription = KafkaInternalFieldDescription.forColumnName(columnHandle.getName()); + switch (fieldDescription) { + case SEGMENT_COUNT_FIELD: + currentRowValuesMap.put(columnHandle, longValueProvider(totalMessages)); + break; + case PARTITION_OFFSET_FIELD: + currentRowValuesMap.put(columnHandle, longValueProvider(messageAndOffset.offset())); + break; + case MESSAGE_FIELD: + currentRowValuesMap.put(columnHandle, bytesValueProvider(messageData)); + break; + case MESSAGE_LENGTH_FIELD: + currentRowValuesMap.put(columnHandle, longValueProvider(messageData.length)); + break; + case KEY_FIELD: + currentRowValuesMap.put(columnHandle, bytesValueProvider(keyData)); + break; + case KEY_LENGTH_FIELD: + currentRowValuesMap.put(columnHandle, longValueProvider(keyData.length)); + break; + case KEY_CORRUPT_FIELD: + currentRowValuesMap.put(columnHandle, booleanValueProvider(!decodedKey.isPresent())); + break; + case MESSAGE_CORRUPT_FIELD: + currentRowValuesMap.put(columnHandle, booleanValueProvider(!decodedValue.isPresent())); + break; + case PARTITION_ID_FIELD: + currentRowValuesMap.put(columnHandle, longValueProvider(split.getPartitionId())); + break; + case SEGMENT_START_FIELD: + currentRowValuesMap.put(columnHandle, longValueProvider(startTs)); + break; + case SEGMENT_END_FIELD: + currentRowValuesMap.put(columnHandle, longValueProvider(endTs)); + break; + case OFFSET_TIMESTAMP_FIELD: + currentRowValuesMap.put(columnHandle, longValueProvider(populateOffsetTimestamp(startTs, endTs))); + default: + throw new IllegalArgumentException("unknown internal field " + fieldDescription); + } + } + } + + decodedKey.ifPresent(currentRowValuesMap::putAll); + decodedValue.ifPresent(currentRowValuesMap::putAll); + + for (int i = 0; i < columnHandles.size(); i++) { + ColumnHandle columnHandle = columnHandles.get(i); + currentRowValues[i] = currentRowValuesMap.get(columnHandle); + } + + return true; // Advanced successfully. + } + + private void openFetchRequest() + { + if (messageAndOffsetIterator == null) { + log.info("Fetching %d bytes from partition %d @offset %d (%d - %d) -- %d messages read so far", + fetchSize, split.getPartitionId(), cursorOffset, split.getStart(), split.getEnd(), totalMessages); + cursorOffset += fetchedSize; + if (cursorOffset < split.getEnd()) { + FetchRequest req = new FetchRequest(split.getTopicName(), split.getPartitionId(), cursorOffset, fetchSize); + SimpleConsumer consumer = consumerManager.getConsumer(split.getLeader()); + + ByteBufferMessageSet fetch = consumer.fetch(req); + log.debug("\t...fetched %s bytes, validBytes=%s, initialOffset=%s", fetch.sizeInBytes(), fetch.validBytes(), fetch.getInitialOffset()); + int errorCode = fetch.getErrorCode(); + if (errorCode != ErrorMapping.NoError() && errorCode != ErrorMapping.OffsetOutOfRangeCode()) { + log.warn("Fetch response has error: %d", errorCode); + throw new PrestoException(KAFKA_SPLIT_ERROR, "could not fetch data from Kafka, error code is '" + errorCode + "'"); + } + + fetchedSize = fetch.validBytes(); + messageAndOffsetIterator = fetch.iterator(); + } + } + } + + private long populateOffsetTimestamp(long startTs, long endTs) + { + if (startTs == OffsetRequest.EarliestTime()) { + startTs = 0; + } + + if (endTs == OffsetRequest.LatestTime()) { + endTs = System.currentTimeMillis(); + } + + return startTs + (endTs - startTs) / 2; + } + + @Override + public boolean getBoolean(int field) + { + return getFieldValueProvider(field, boolean.class).getBoolean(); + } + + @Override + public long getLong(int field) + { + return getFieldValueProvider(field, long.class).getLong(); + } + + @Override + public double getDouble(int field) + { + return getFieldValueProvider(field, double.class).getDouble(); + } + + @Override + public Slice getSlice(int field) + { + return getFieldValueProvider(field, Slice.class).getSlice(); + } + + @Override + public Object getObject(int field) + { + return getFieldValueProvider(field, Block.class).getBlock(); + } + + @Override + public boolean isNull(int field) + { + checkArgument(field < columnHandles.size(), "Invalid field index"); + return currentRowValues[field] == null || currentRowValues[field].isNull(); + } + + private FieldValueProvider getFieldValueProvider(int field, Class expectedType) + { + checkArgument(field < columnHandles.size(), "Invalid field index"); + checkFieldType(field, expectedType); + return currentRowValues[field]; + } + + private void checkFieldType(int field, Class expected) + { + Class actual = getType(field).getJavaType(); + checkArgument(actual == expected, "Expected field %s to be type %s but is %s", field, expected, actual); + } + + @Override + public void close() + { + } + } +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaRecordSetProvider.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaRecordSetProvider.java new file mode 100644 index 0000000000000..b926e1c44323b --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaRecordSetProvider.java @@ -0,0 +1,80 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.decoder.DispatchingRowDecoderFactory; +import com.facebook.presto.decoder.RowDecoder; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.RecordSet; +import com.facebook.presto.spi.connector.ConnectorRecordSetProvider; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.google.common.collect.ImmutableList; + +import javax.inject.Inject; + +import java.util.List; + +import static com.facebook.presto.kafka.KafkaHandleResolver.convertSplit; +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static java.util.Collections.emptyMap; +import static java.util.Objects.requireNonNull; + +/** + * Factory for Kafka specific {@link RecordSet} instances. + */ +public class KafkaRecordSetProvider + implements ConnectorRecordSetProvider +{ + private final KafkaSimpleConsumerManager consumerManager; + private final DispatchingRowDecoderFactory decoderFactory; + private final KafkaConnectorConfig config; + + @Inject + public KafkaRecordSetProvider(DispatchingRowDecoderFactory decoderFactory, KafkaSimpleConsumerManager consumerManager, KafkaConnectorConfig config) + { + this.decoderFactory = requireNonNull(decoderFactory, "decoderFactory is null"); + this.consumerManager = requireNonNull(consumerManager, "consumerManager is null"); + this.config = requireNonNull(config, "config is null"); + } + + @Override + public RecordSet getRecordSet(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorSplit split, List columns) + { + KafkaSplit kafkaSplit = convertSplit(split); + + List kafkaColumns = columns.stream() + .map(KafkaHandleResolver::convertColumnHandle) + .collect(ImmutableList.toImmutableList()); + + RowDecoder keyDecoder = decoderFactory.create( + kafkaSplit.getKeyDataFormat(), + emptyMap(), + kafkaColumns.stream() + .filter(col -> !col.isInternal()) + .filter(KafkaColumnHandle::isKeyDecoder) + .collect(toImmutableSet())); + + RowDecoder messageDecoder = decoderFactory.create( + kafkaSplit.getMessageDataFormat(), + emptyMap(), + kafkaColumns.stream() + .filter(col -> !col.isInternal()) + .filter(col -> !col.isKeyDecoder()) + .collect(toImmutableSet())); + + return new KafkaRecordSet(kafkaSplit, consumerManager, kafkaColumns, keyDecoder, messageDecoder, config.getFetchSize()); + } +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaSimpleConsumerManager.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaSimpleConsumerManager.java new file mode 100644 index 0000000000000..53756c614efa7 --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaSimpleConsumerManager.java @@ -0,0 +1,103 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.spi.HostAddress; +import com.facebook.presto.spi.NodeManager; +import com.google.common.base.Throwables; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import io.airlift.log.Logger; +import kafka.javaapi.consumer.SimpleConsumer; + +import javax.annotation.PreDestroy; +import javax.inject.Inject; + +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import static java.lang.Math.toIntExact; +import static java.util.Objects.requireNonNull; + +/** + * Manages connections to the Kafka nodes. A worker may connect to multiple Kafka nodes depending on the segments and partitions + * it needs to process. According to the Kafka source code, a Kafka {@link kafka.javaapi.consumer.SimpleConsumer} is thread-safe. + */ +public class KafkaSimpleConsumerManager +{ + private static final Logger log = Logger.get(KafkaSimpleConsumerManager.class); + + private final LoadingCache consumerCache; + + private final String connectorId; + private final NodeManager nodeManager; + private final int connectTimeoutMillis; + private final int bufferSizeBytes; + + @Inject + public KafkaSimpleConsumerManager( + KafkaConnectorId connectorId, + KafkaConnectorConfig kafkaConnectorConfig, + NodeManager nodeManager) + { + this.connectorId = requireNonNull(connectorId, "connectorId is null").toString(); + this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); + + requireNonNull(kafkaConnectorConfig, "kafkaConfig is null"); + this.connectTimeoutMillis = toIntExact(kafkaConnectorConfig.getKafkaConnectTimeout().toMillis()); + this.bufferSizeBytes = toIntExact(kafkaConnectorConfig.getKafkaBufferSize().toBytes()); + + this.consumerCache = CacheBuilder.newBuilder().build(new SimpleConsumerCacheLoader()); + } + + @PreDestroy + public void tearDown() + { + for (Map.Entry entry : consumerCache.asMap().entrySet()) { + try { + entry.getValue().close(); + } + catch (Exception e) { + log.warn(e, "While closing consumer %s:", entry.getKey()); + } + } + } + + public SimpleConsumer getConsumer(HostAddress host) + { + requireNonNull(host, "host is null"); + try { + return consumerCache.get(host); + } + catch (ExecutionException e) { + throw Throwables.propagate(e.getCause()); + } + } + + private class SimpleConsumerCacheLoader + extends CacheLoader + { + @Override + public SimpleConsumer load(HostAddress host) + throws Exception + { + log.debug("Creating new Consumer for %s", host); + return new SimpleConsumer(host.getHostText(), + host.getPort(), + connectTimeoutMillis, + bufferSizeBytes); + } + } +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaSplit.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaSplit.java new file mode 100644 index 0000000000000..17edf7d47e6e8 --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaSplit.java @@ -0,0 +1,165 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.HostAddress; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +/** + * Represents a kafka specific {@link ConnectorSplit}. Each split is mapped to a segment file on disk (based off the segment offset start() and end() values) so that + * a partition can be processed by reading segment files from partition leader. Otherwise, a Kafka topic could only be processed along partition boundaries. + *

+ * When planning to process a Kafka topic with Presto, using smaller than the recommended segment size (default is 1G) allows Presto to optimize early and process a topic + * with more workers in parallel. + */ +public class KafkaSplit + implements ConnectorSplit +{ + private final String connectorId; + private final String topicName; + private final String keyDataFormat; + private final String messageDataFormat; + private final int partitionId; + private final long start; + private final long end; + private final HostAddress leader; + private final long startTs; + private final long endTs; + + @JsonCreator + public KafkaSplit( + @JsonProperty("connectorId") String connectorId, + @JsonProperty("topicName") String topicName, + @JsonProperty("keyDataFormat") String keyDataFormat, + @JsonProperty("messageDataFormat") String messageDataFormat, + @JsonProperty("partitionId") int partitionId, + @JsonProperty("start") long start, + @JsonProperty("end") long end, + @JsonProperty("leader") HostAddress leader, + @JsonProperty("startTs") long startTs, + @JsonProperty("endTs") long endTs) + { + this.connectorId = requireNonNull(connectorId, "connector id is null"); + this.topicName = requireNonNull(topicName, "topicName is null"); + this.keyDataFormat = requireNonNull(keyDataFormat, "dataFormat is null"); + this.messageDataFormat = requireNonNull(messageDataFormat, "messageDataFormat is null"); + this.partitionId = partitionId; + this.start = start; + this.end = end; + this.leader = requireNonNull(leader, "leader address is null"); + this.startTs = startTs; + this.endTs = endTs; + } + + @JsonProperty + public String getConnectorId() + { + return connectorId; + } + + @JsonProperty + public long getStart() + { + return start; + } + + @JsonProperty + public long getEnd() + { + return end; + } + + @JsonProperty + public String getTopicName() + { + return topicName; + } + + @JsonProperty + public String getKeyDataFormat() + { + return keyDataFormat; + } + + @JsonProperty + public String getMessageDataFormat() + { + return messageDataFormat; + } + + @JsonProperty + public int getPartitionId() + { + return partitionId; + } + + @JsonProperty + public HostAddress getLeader() + { + return leader; + } + + @JsonProperty + public long getStartTs() + { + return startTs; + } + + @JsonProperty + public long getEndTs() + { + return endTs; + } + + @Override + public boolean isRemotelyAccessible() + { + return true; + } + + @Override + public List getAddresses() + { + return ImmutableList.of(leader); + } + + @Override + public Object getInfo() + { + return this; + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("connectorId", connectorId) + .add("topicName", topicName) + .add("keyDataFormat", keyDataFormat) + .add("messageDataFormat", messageDataFormat) + .add("partitionId", partitionId) + .add("start", start) + .add("end", end) + .add("leader", leader) + .toString(); + } +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaSplitManager.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaSplitManager.java new file mode 100644 index 0000000000000..056bfea7e2b29 --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaSplitManager.java @@ -0,0 +1,153 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.ConnectorSplitSource; +import com.facebook.presto.spi.ConnectorTableLayoutHandle; +import com.facebook.presto.spi.FixedSplitSource; +import com.facebook.presto.spi.HostAddress; +import com.facebook.presto.spi.connector.ConnectorSplitManager; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.google.common.collect.ImmutableList; +import io.airlift.log.Logger; +import kafka.api.OffsetRequest; +import kafka.cluster.Broker; +import kafka.cluster.Cluster; +import kafka.cluster.Partition; +import kafka.javaapi.consumer.SimpleConsumer; +import kafka.utils.ZkUtils; +import org.I0Itec.zkclient.ZkClient; + +import javax.inject.Inject; + +import java.util.List; + +import static com.facebook.presto.kafka.KafkaHandleResolver.convertLayout; +import static java.util.Objects.requireNonNull; + +/** + * Kafka specific implementation of {@link ConnectorSplitManager}. + */ +public class KafkaSplitManager + implements ConnectorSplitManager +{ + private static final Logger log = Logger.get(KafkaSplitManager.class); + + private final String connectorId; + private final KafkaSimpleConsumerManager consumerManager; + private final KafkaConnectorConfig config; + + @Inject + public KafkaSplitManager( + KafkaConnectorId connectorId, + KafkaConnectorConfig kafkaConnectorConfig, + KafkaSimpleConsumerManager consumerManager) + { + this.connectorId = requireNonNull(connectorId, "connectorId is null").toString(); + this.consumerManager = requireNonNull(consumerManager, "consumerManager is null"); + + requireNonNull(kafkaConnectorConfig, "kafkaConfig is null"); + this.config = kafkaConnectorConfig; + } + + @Override + public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableLayoutHandle layout, SplitSchedulingStrategy splitSchedulingStrategy) + { + KafkaTableHandle kafkaTableHandle = convertLayout(layout).getTable(); + ZkClient zkClient = KafkaUtil.newZkClient(config.getZkEndpoint()); + + ImmutableList.Builder splits = ImmutableList.builder(); + Cluster cluster = ZkUtils.getCluster(zkClient); + List partitions = KafkaUtil.getPartitionsForTopic(zkClient, kafkaTableHandle.getTopicName()); + + long estimatedTotalSize = 0L; + + for (Partition part : partitions) { + log.debug("Adding Partition %s/%s from broker %s", kafkaTableHandle.getTopicName(), part.partId(), part.brokerId()); + Broker leader = cluster.getBroker(part.brokerId()).get(); + + if (leader == null) { // Leader election going on... + log.error("No leader for partition %s/%s found!", kafkaTableHandle.getTopicName(), part.partId()); + continue; + } + + HostAddress partitionLeader = HostAddress.fromParts(leader.host(), leader.port()); + + SimpleConsumer leaderConsumer = consumerManager.getConsumer(partitionLeader); + // Kafka contains a reverse list of "end - start" pairs for the splits + + KafkaTableLayoutHandle layoutHandle = (KafkaTableLayoutHandle) layout; + long startTs = layoutHandle.getOffsetStartTs(); + long endTs = layoutHandle.getOffsetEndTs(); + + long[] offsets = findAllOffsets(leaderConsumer, kafkaTableHandle.getTopicName(), part.partId(), startTs, endTs); + for (int i = offsets.length - 1; i > 0; i--) { + KafkaSplit split = new KafkaSplit( + connectorId, + kafkaTableHandle.getTopicName(), + kafkaTableHandle.getKeyDataFormat(), + kafkaTableHandle.getMessageDataFormat(), + part.partId(), + offsets[i], + offsets[i - 1], + partitionLeader, + startTs, + endTs); + splits.add(split); + + long splitSize = (split.getEnd() - split.getStart()) / 1024 / 1024; + log.debug("Split summarize: %s-%s (%sMB)", split.getStart(), split.getEnd(), splitSize); + estimatedTotalSize += splitSize; + } + } + + ImmutableList builtSplits = splits.build(); + log.info("Built " + builtSplits.size() + " splits"); + + log.info("EstimatedTotalSize: %s", estimatedTotalSize); + return new FixedSplitSource(builtSplits); + } + + private static long[] findAllOffsets(SimpleConsumer consumer, String topicName, int partitionId, long startTs, long endTs) + { + // startTs: start timestamp, or -2/null as earliest + // endTs: end timestamp, or -1/null as latest + if (startTs >= endTs && endTs != OffsetRequest.LatestTime()) { + throw new IllegalArgumentException(String.format("Invalid Kafka Offset start/end pair: %s - %s", startTs, endTs)); + } + + long[] offsetsBeforeStartTs = consumer.getOffsetsBefore(topicName, partitionId, startTs, Integer.MAX_VALUE); + long[] offsetsBeforeEndTs = consumer.getOffsetsBefore(topicName, partitionId, endTs, Integer.MAX_VALUE); + log.debug("NumOffsetsBeforeStartTs=%s, NumOffsetsBeforeEndTs=%s", offsetsBeforeStartTs.length, offsetsBeforeEndTs.length); + + if (offsetsBeforeStartTs.length == 0) { + return offsetsBeforeEndTs; + } + + long[] offsets = new long[offsetsBeforeEndTs.length - offsetsBeforeStartTs.length + 1]; + long startOffset = offsetsBeforeStartTs[0]; + + for (int i = 0; i < offsetsBeforeEndTs.length; i++) { + if (offsetsBeforeEndTs[i] == startOffset) { + offsets[i] = startOffset; + break; + } + offsets[i] = offsetsBeforeEndTs[i]; + } + + return offsets; + } +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaTableDescriptionSupplier.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaTableDescriptionSupplier.java new file mode 100644 index 0000000000000..2fcdbe539ef47 --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaTableDescriptionSupplier.java @@ -0,0 +1,138 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.decoder.dummy.DummyRowDecoder; +import com.facebook.presto.spi.SchemaTableName; +import com.google.common.base.Splitter; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import io.airlift.json.JsonCodec; +import io.airlift.log.Logger; + +import javax.inject.Inject; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Strings.isNullOrEmpty; +import static java.nio.file.Files.readAllBytes; +import static java.util.Arrays.asList; +import static java.util.Objects.requireNonNull; + +public class KafkaTableDescriptionSupplier + implements Supplier> +{ + private static final Logger log = Logger.get(KafkaTableDescriptionSupplier.class); + + private final JsonCodec topicDescriptionCodec; + private final File tableDescriptionDir; + private final String defaultSchema; + private final Set tableNames; + + @Inject + KafkaTableDescriptionSupplier(KafkaConnectorConfig kafkaConnectorConfig, + JsonCodec topicDescriptionCodec) + { + this.topicDescriptionCodec = requireNonNull(topicDescriptionCodec, "topicDescriptionCodec is null"); + + requireNonNull(kafkaConnectorConfig, "kafkaConfig is null"); + this.tableDescriptionDir = kafkaConnectorConfig.getTableDescriptionDir(); + this.defaultSchema = kafkaConnectorConfig.getDefaultSchema(); + this.tableNames = ImmutableSet.copyOf(kafkaConnectorConfig.getTableNames()); + } + + @Override + public Map get() + { + ImmutableMap.Builder builder = ImmutableMap.builder(); + + log.debug("Loading kafka table definitions from %s", tableDescriptionDir.getAbsolutePath()); + + try { + for (File file : listFiles(tableDescriptionDir)) { + if (file.isFile() && file.getName().endsWith(".json")) { + KafkaTopicDescription table = topicDescriptionCodec.fromJson(readAllBytes(file.toPath())); + String schemaName = firstNonNull(table.getSchemaName(), defaultSchema); + log.debug("Kafka table %s.%s: %s", schemaName, table.getTableName(), table); + builder.put(new SchemaTableName(schemaName, table.getTableName()), table); + } + } + + Map tableDefinitions = builder.build(); + + log.debug("Loaded Table definitions: %s", tableDefinitions.keySet()); + + builder = ImmutableMap.builder(); + for (String definedTable : tableNames) { + SchemaTableName tableName; + try { + tableName = parseTableName(definedTable); + } + catch (IllegalArgumentException iae) { + tableName = new SchemaTableName(defaultSchema, definedTable); + } + + if (tableDefinitions.containsKey(tableName)) { + KafkaTopicDescription kafkaTable = tableDefinitions.get(tableName); + log.debug("Found Table definition for %s: %s", tableName, kafkaTable); + builder.put(tableName, kafkaTable); + } + else { + // A dummy table definition only supports the internal columns. + log.debug("Created dummy Table definition for %s", tableName); + builder.put(tableName, new KafkaTopicDescription(tableName.getTableName(), + tableName.getSchemaName(), + definedTable, + new KafkaTopicFieldGroup(DummyRowDecoder.NAME, ImmutableList.of()), + new KafkaTopicFieldGroup(DummyRowDecoder.NAME, ImmutableList.of()))); + } + } + + return builder.build(); + } + catch (IOException e) { + log.warn(e, "Error: "); + throw Throwables.propagate(e); + } + } + + private static List listFiles(File dir) + { + if ((dir != null) && dir.isDirectory()) { + File[] files = dir.listFiles(); + if (files != null) { + log.debug("Considering files: %s", asList(files)); + return ImmutableList.copyOf(files); + } + } + return ImmutableList.of(); + } + + private static SchemaTableName parseTableName(String schemaTableName) + { + checkArgument(!isNullOrEmpty(schemaTableName), "schemaTableName is null or is empty"); + List parts = Splitter.on('.').splitToList(schemaTableName); + checkArgument(parts.size() == 2, "Invalid schemaTableName: %s", schemaTableName); + return new SchemaTableName(parts.get(0), parts.get(1)); + } +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaTableHandle.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaTableHandle.java new file mode 100644 index 0000000000000..8475b2c68c98f --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaTableHandle.java @@ -0,0 +1,151 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.spi.ConnectorTableHandle; +import com.facebook.presto.spi.SchemaTableName; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +/** + * Kafka specific {@link ConnectorTableHandle}. + */ +public final class KafkaTableHandle + implements ConnectorTableHandle +{ + /** + * connector id + */ + private final String connectorId; + + /** + * The schema name for this table. Is set through configuration and read + * using {@link KafkaConnectorConfig#getDefaultSchema()}. Usually 'default'. + */ + private final String schemaName; + + /** + * The table name used by presto. + */ + private final String tableName; + + /** + * The topic name that is read from Kafka. + */ + private final String topicName; + + private final String keyDataFormat; + private final String messageDataFormat; + + @JsonCreator + public KafkaTableHandle( + @JsonProperty("connectorId") String connectorId, + @JsonProperty("schemaName") String schemaName, + @JsonProperty("tableName") String tableName, + @JsonProperty("topicName") String topicName, + @JsonProperty("keyDataFormat") String keyDataFormat, + @JsonProperty("messageDataFormat") String messageDataFormat) + { + this.connectorId = requireNonNull(connectorId, "connectorId is null"); + this.schemaName = requireNonNull(schemaName, "schemaName is null"); + this.tableName = requireNonNull(tableName, "tableName is null"); + this.topicName = requireNonNull(topicName, "topicName is null"); + this.keyDataFormat = requireNonNull(keyDataFormat, "keyDataFormat is null"); + this.messageDataFormat = requireNonNull(messageDataFormat, "messageDataFormat is null"); + } + + @JsonProperty + public String getConnectorId() + { + return connectorId; + } + + @JsonProperty + public String getSchemaName() + { + return schemaName; + } + + @JsonProperty + public String getTableName() + { + return tableName; + } + + @JsonProperty + public String getTopicName() + { + return topicName; + } + + @JsonProperty + public String getKeyDataFormat() + { + return keyDataFormat; + } + + @JsonProperty + public String getMessageDataFormat() + { + return messageDataFormat; + } + + public SchemaTableName toSchemaTableName() + { + return new SchemaTableName(schemaName, tableName); + } + + @Override + public int hashCode() + { + return Objects.hash(connectorId, schemaName, tableName, topicName, keyDataFormat, messageDataFormat); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + KafkaTableHandle other = (KafkaTableHandle) obj; + return Objects.equals(this.connectorId, other.connectorId) + && Objects.equals(this.schemaName, other.schemaName) + && Objects.equals(this.tableName, other.tableName) + && Objects.equals(this.topicName, other.topicName) + && Objects.equals(this.keyDataFormat, other.keyDataFormat) + && Objects.equals(this.messageDataFormat, other.messageDataFormat); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("connectorId", connectorId) + .add("schemaName", schemaName) + .add("tableName", tableName) + .add("topicName", topicName) + .add("keyDataFormat", keyDataFormat) + .add("messageDataFormat", messageDataFormat) + .toString(); + } +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaTableLayoutHandle.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaTableLayoutHandle.java new file mode 100644 index 0000000000000..57e6ffc6e6153 --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaTableLayoutHandle.java @@ -0,0 +1,64 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.spi.ConnectorTableLayoutHandle; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import kafka.api.OffsetRequest; + +import static java.util.Objects.requireNonNull; + +public class KafkaTableLayoutHandle + implements ConnectorTableLayoutHandle +{ + private final KafkaTableHandle table; + private final long offsetStartTs; + private final long offsetEndTs; + + @JsonCreator + public KafkaTableLayoutHandle( + @JsonProperty("table") KafkaTableHandle table, + @JsonProperty("offset_start_ts") Long offsetStartTs, + @JsonProperty("offset_end_ts") Long offsetEndTs) + { + this.table = requireNonNull(table, "table is null"); + this.offsetStartTs = offsetStartTs == null ? OffsetRequest.EarliestTime() : offsetStartTs; + this.offsetEndTs = offsetEndTs == null ? OffsetRequest.LatestTime() : offsetEndTs; + } + + @JsonProperty + public KafkaTableHandle getTable() + { + return table; + } + + @JsonProperty + public long getOffsetStartTs() + { + return offsetStartTs; + } + + @JsonProperty + public long getOffsetEndTs() + { + return offsetEndTs; + } + + @Override + public String toString() + { + return table.toString(); + } +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaTopicDescription.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaTopicDescription.java new file mode 100644 index 0000000000000..443bb6828940b --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaTopicDescription.java @@ -0,0 +1,92 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Strings.isNullOrEmpty; +import static java.util.Objects.requireNonNull; + +/** + * Json description to parse a row on a Kafka topic. A row contains a message and an optional key. See the documentation for the exact JSON syntax. + */ +public class KafkaTopicDescription +{ + private final String tableName; + private final String topicName; + private final String schemaName; + private final KafkaTopicFieldGroup key; + private final KafkaTopicFieldGroup message; + + @JsonCreator + public KafkaTopicDescription( + @JsonProperty("tableName") String tableName, + @JsonProperty("schemaName") String schemaName, + @JsonProperty("topicName") String topicName, + @JsonProperty("key") KafkaTopicFieldGroup key, + @JsonProperty("message") KafkaTopicFieldGroup message) + { + checkArgument(!isNullOrEmpty(tableName), "tableName is null or is empty"); + this.tableName = tableName; + this.topicName = requireNonNull(topicName, "topicName is null"); + this.schemaName = schemaName; + this.key = key; + this.message = message; + } + + @JsonProperty + public String getTableName() + { + return tableName; + } + + @JsonProperty + public String getTopicName() + { + return topicName; + } + + @JsonProperty + public String getSchemaName() + { + return schemaName; + } + + @JsonProperty + public KafkaTopicFieldGroup getKey() + { + return key; + } + + @JsonProperty + public KafkaTopicFieldGroup getMessage() + { + return message; + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("tableName", tableName) + .add("topicName", topicName) + .add("schemaName", schemaName) + .add("key", key) + .add("message", message) + .toString(); + } +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaTopicFieldDescription.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaTopicFieldDescription.java new file mode 100644 index 0000000000000..38c9861ea5f56 --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaTopicFieldDescription.java @@ -0,0 +1,159 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.type.Type; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Strings.isNullOrEmpty; +import static java.util.Objects.requireNonNull; + +/** + * Json description to parse a single field from a Kafka topic message. See {@link com.facebook.presto.kafka.KafkaTopicDescription} for more details. + */ +public final class KafkaTopicFieldDescription +{ + private final String name; + private final Type type; + private final String mapping; + private final String comment; + private final String dataFormat; + private final String formatHint; + private final boolean hidden; + + @JsonCreator + public KafkaTopicFieldDescription( + @JsonProperty("name") String name, + @JsonProperty("type") Type type, + @JsonProperty("mapping") String mapping, + @JsonProperty("comment") String comment, + @JsonProperty("dataFormat") String dataFormat, + @JsonProperty("formatHint") String formatHint, + @JsonProperty("hidden") boolean hidden) + { + checkArgument(!isNullOrEmpty(name), "name is null or is empty"); + this.name = name; + this.type = requireNonNull(type, "type is null"); + this.mapping = mapping; + this.comment = comment; + this.dataFormat = dataFormat; + this.formatHint = formatHint; + this.hidden = hidden; + } + + @JsonProperty + public String getName() + { + return name; + } + + @JsonProperty + public Type getType() + { + return type; + } + + @JsonProperty + public String getMapping() + { + return mapping; + } + + @JsonProperty + public String getComment() + { + return comment; + } + + @JsonProperty + public String getDataFormat() + { + return dataFormat; + } + + @JsonProperty + public String getFormatHint() + { + return formatHint; + } + + @JsonProperty + public boolean isHidden() + { + return hidden; + } + + KafkaColumnHandle getColumnHandle(String connectorId, boolean keyDecoder, int index) + { + return new KafkaColumnHandle(connectorId, + index, + getName(), + getType(), + getMapping(), + getDataFormat(), + getFormatHint(), + keyDecoder, + isHidden(), + false); + } + + ColumnMetadata getColumnMetadata() + { + return new ColumnMetadata(getName(), getType(), getComment(), isHidden()); + } + + @Override + public int hashCode() + { + return Objects.hash(name, type, mapping, dataFormat, formatHint, hidden); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + KafkaTopicFieldDescription other = (KafkaTopicFieldDescription) obj; + return Objects.equals(this.name, other.name) && + Objects.equals(this.type, other.type) && + Objects.equals(this.mapping, other.mapping) && + Objects.equals(this.dataFormat, other.dataFormat) && + Objects.equals(this.formatHint, other.formatHint) && + Objects.equals(this.hidden, other.hidden); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("name", name) + .add("type", type) + .add("mapping", mapping) + .add("dataFormat", dataFormat) + .add("formatHint", formatHint) + .add("hidden", hidden) + .toString(); + } +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaTopicFieldGroup.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaTopicFieldGroup.java new file mode 100644 index 0000000000000..4b5384fe982e1 --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaTopicFieldGroup.java @@ -0,0 +1,62 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +/** + * Groups the field descriptions for message or key. + */ +public class KafkaTopicFieldGroup +{ + private final String dataFormat; + private final List fields; + + @JsonCreator + public KafkaTopicFieldGroup( + @JsonProperty("dataFormat") String dataFormat, + @JsonProperty("fields") List fields) + { + this.dataFormat = requireNonNull(dataFormat, "dataFormat is null"); + this.fields = ImmutableList.copyOf(requireNonNull(fields, "fields is null")); + } + + @JsonProperty + public String getDataFormat() + { + return dataFormat; + } + + @JsonProperty + public List getFields() + { + return fields; + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("dataFormat", dataFormat) + .add("fields", fields) + .toString(); + } +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaTransactionHandle.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaTransactionHandle.java new file mode 100644 index 0000000000000..f777be380ef2c --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaTransactionHandle.java @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; + +public enum KafkaTransactionHandle + implements ConnectorTransactionHandle +{ + INSTANCE +} diff --git a/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaUtil.java b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaUtil.java new file mode 100644 index 0000000000000..4edf4f6da06fd --- /dev/null +++ b/presto-kafka07/src/main/java/com/facebook/presto/kafka/KafkaUtil.java @@ -0,0 +1,159 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.kafka; + +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; +import kafka.cluster.Broker; +import kafka.cluster.Partition; +import kafka.consumer.ConsumerConfig; +import kafka.consumer.SimpleConsumer; +import kafka.utils.ZKConfig; +import kafka.utils.ZkUtils; +import org.I0Itec.zkclient.ZkClient; +import scala.collection.JavaConversions; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +/** + * A collection of utility methods for accessing Kafka. + * + * @author Raghu Angadi + */ +public final class KafkaUtil +{ + public static final ConsumerConfig DEFAULT_CONSUMER_CONFIG; + + static { + Properties properties = new Properties(); + properties.setProperty("groupid", "this-should-not-be-used"); + DEFAULT_CONSUMER_CONFIG = new ConsumerConfig(properties); + } + + private KafkaUtil() + { + } + + /** + * create ZkClient with default options. + */ + public static ZkClient newZkClient(String zkConnect) + { + // get defaults from ZkConfig. + ZKConfig config = new ZKConfig(new Properties()); + + return new ZkClient(zkConnect, + config.zkSessionTimeoutMs(), + config.zkConnectionTimeoutMs(), + kafka.utils.ZKStringSerializer$.MODULE$); + } + + /** + * Returns partitions for given topic. An empty list if the topic is not + * found. + */ + public static List getPartitionsForTopic(ZkClient zkClient, + String topic) + { + // handle scala <-> java conversions. + scala.collection.Iterator topics = + JavaConversions.asScalaIterator(Iterators.forArray(topic)); + Map> map = + JavaConversions.mapAsJavaMap(ZkUtils.getPartitionsForTopics(zkClient, topics)); + + // since we are asking for just one topic, map's size is 0 or 1. + if (map.size() > 0) { + List partitions = JavaConversions.seqAsJavaList( + map.values().iterator().next()); + // transform string to Partition object + return Lists.newArrayList( + Lists.transform(partitions, + input -> Partition.parse(input))); + } + + return new ArrayList<>(); + } + + /** + * Returns latest offset before the given timestamp. If there is no offset + * avaliable, returns the earliest available. An offset before the timestamp + * may not be available if the messages are already rotated. + */ + public static long getBeforeOrEarliestOffset(SimpleConsumer consumer, + String topic, + int partId, + long time) + { + long[] offsets = consumer.getOffsetsBefore(topic, partId, time, 1); + if (offsets.length == 0) { + // then the earliest offset + offsets = consumer.getOffsetsBefore(topic, partId, -2, 1); + } + + return (offsets.length > 0) ? offsets[0] : 0; + } + + /** + * Returns the topics on given Kafka Server + */ + public static List getTopics(ZkClient zkClient) + { + String topicPath = ZkUtils.BrokerTopicsPath(); + return zkClient.getChildren(topicPath); + } + + /** + * Returns the brokers currently registered + */ + public static List getBrokersIds(ZkClient zkClient) + { + String brokerPath = ZkUtils.BrokerIdsPath(); + List brokers = zkClient.getChildren(brokerPath); + List brokerIds = new ArrayList(); + for (String s : brokers) { + Long l = Long.parseLong(s); + brokerIds.add(l); + } + + return brokerIds; + } + + /** + * Returns the number of partitions for a given topic and broker. + */ + public static Integer getNumPartitions(ZkClient zkClient, String topic, Long broker) + { + String topicPath = ZkUtils.BrokerTopicsPath(); + String partitionPath = topicPath + "/" + topic + "/" + broker.toString(); + + String numPartitions = zkClient.readData(partitionPath, true); + if (numPartitions == null) { + return 0; + } + else { + return Integer.parseInt(numPartitions); + } + } + + public static SimpleConsumer newSimpleConsumer(Broker broker) + { + return new SimpleConsumer(broker.host(), broker.port(), + DEFAULT_CONSUMER_CONFIG.socketTimeoutMs(), + DEFAULT_CONSUMER_CONFIG.socketBufferSize()); + } +} diff --git a/presto-kafka07/src/test/java/com/facebook/presto/kafka/KafkaQueryRunner.java b/presto-kafka07/src/test/java/com/facebook/presto/kafka/KafkaQueryRunner.java new file mode 100644 index 0000000000000..1f760f8a4c46d --- /dev/null +++ b/presto-kafka07/src/test/java/com/facebook/presto/kafka/KafkaQueryRunner.java @@ -0,0 +1,142 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.Session; +import com.facebook.presto.kafka.util.CodecSupplier; +import com.facebook.presto.kafka.util.EmbeddedKafka; +import com.facebook.presto.kafka.util.TestUtils; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.metadata.QualifiedObjectName; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.tests.DistributedQueryRunner; +import com.facebook.presto.tests.TestingPrestoClient; +import com.facebook.presto.tpch.TpchPlugin; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.airlift.json.JsonCodec; +import io.airlift.log.Logger; +import io.airlift.log.Logging; +import io.airlift.tpch.TpchTable; + +import java.util.Map; + +import static com.facebook.presto.kafka.util.TestUtils.installKafkaPlugin; +import static com.facebook.presto.kafka.util.TestUtils.loadTpchTopicDescription; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; +import static io.airlift.testing.Closeables.closeAllSuppress; +import static io.airlift.units.Duration.nanosSince; +import static java.util.Locale.ENGLISH; +import static java.util.concurrent.TimeUnit.SECONDS; + +public final class KafkaQueryRunner +{ + private KafkaQueryRunner() + { + } + + private static final Logger log = Logger.get("TestQueries"); + private static final String TPCH_SCHEMA = "tpch"; + + public static DistributedQueryRunner createKafkaQueryRunner(EmbeddedKafka embeddedKafka, TpchTable... tables) + throws Exception + { + return createKafkaQueryRunner(embeddedKafka, ImmutableList.copyOf(tables)); + } + + public static DistributedQueryRunner createKafkaQueryRunner(EmbeddedKafka embeddedKafka, Iterable> tables) + throws Exception + { + DistributedQueryRunner queryRunner = null; + try { + queryRunner = new DistributedQueryRunner(createSession(), 2); + + queryRunner.installPlugin(new TpchPlugin()); + queryRunner.createCatalog("tpch", "tpch"); + + embeddedKafka.start(); + + for (TpchTable table : tables) { + embeddedKafka.createTopics(kafkaTopicName(table)); + } + + Map topicDescriptions = createTpchTopicDescriptions(queryRunner.getCoordinator().getMetadata(), tables); + + installKafkaPlugin(embeddedKafka, queryRunner, topicDescriptions); + + TestingPrestoClient prestoClient = queryRunner.getClient(); + + log.info("Loading data..."); + long startTime = System.nanoTime(); + for (TpchTable table : tables) { + loadTpchTopic(embeddedKafka, prestoClient, table); + } + log.info("Loading complete in %s", nanosSince(startTime).toString(SECONDS)); + + return queryRunner; + } + catch (Throwable e) { + closeAllSuppress(e, queryRunner, embeddedKafka); + throw e; + } + } + + private static void loadTpchTopic(EmbeddedKafka embeddedKafka, TestingPrestoClient prestoClient, TpchTable table) + { + long start = System.nanoTime(); + log.info("Running import for %s", table.getTableName()); + TestUtils.loadTpchTopic(embeddedKafka, prestoClient, kafkaTopicName(table), new QualifiedObjectName("tpch", TINY_SCHEMA_NAME, table.getTableName().toLowerCase(ENGLISH))); + log.info("Imported %s in %s", 0, table.getTableName(), nanosSince(start).convertToMostSuccinctTimeUnit()); + } + + private static String kafkaTopicName(TpchTable table) + { + return TPCH_SCHEMA + "." + table.getTableName().toLowerCase(ENGLISH); + } + + private static Map createTpchTopicDescriptions(Metadata metadata, Iterable> tables) + throws Exception + { + JsonCodec topicDescriptionJsonCodec = new CodecSupplier<>(KafkaTopicDescription.class, metadata).get(); + + ImmutableMap.Builder topicDescriptions = ImmutableMap.builder(); + for (TpchTable table : tables) { + String tableName = table.getTableName(); + SchemaTableName tpchTable = new SchemaTableName(TPCH_SCHEMA, tableName); + + topicDescriptions.put(loadTpchTopicDescription(topicDescriptionJsonCodec, tpchTable.toString(), tpchTable)); + } + return topicDescriptions.build(); + } + + public static Session createSession() + { + return testSessionBuilder() + .setCatalog("kafka") + .setSchema(TPCH_SCHEMA) + .build(); + } + + public static void main(String[] args) + throws Exception + { + Logging.initialize(); + DistributedQueryRunner queryRunner = createKafkaQueryRunner(EmbeddedKafka.createEmbeddedKafka(), TpchTable.getTables()); + Thread.sleep(10); + Logger log = Logger.get(KafkaQueryRunner.class); + log.info("======== SERVER STARTED ========"); + log.info("\n====\n%s\n====", queryRunner.getCoordinator().getBaseUrl()); + } +} diff --git a/presto-kafka07/src/test/java/com/facebook/presto/kafka/TestKafkaConnectorConfig.java b/presto-kafka07/src/test/java/com/facebook/presto/kafka/TestKafkaConnectorConfig.java new file mode 100644 index 0000000000000..e824778a9beeb --- /dev/null +++ b/presto-kafka07/src/test/java/com/facebook/presto/kafka/TestKafkaConnectorConfig.java @@ -0,0 +1,68 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.google.common.collect.ImmutableMap; +import io.airlift.configuration.testing.ConfigAssertions; +import org.testng.annotations.Test; + +import java.io.File; +import java.util.Map; + +public class TestKafkaConnectorConfig +{ + @Test + public void testDefaults() + { + ConfigAssertions.assertRecordedDefaults(ConfigAssertions.recordDefaults(KafkaConnectorConfig.class) + .setNodes("") + .setKafkaConnectTimeout("10s") + .setKafkaBufferSize("64kB") + .setDefaultSchema("default") + .setTableNames("") + .setTableDescriptionDir(new File("etc/kafka07/")) + .setHideInternalColumns(true) + .setFetchSize(10485760) + .setZkEndpoint("")); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("kafka.table-description-dir", "/var/lib/kafka") + .put("kafka.table-names", "table1, table2, table3") + .put("kafka.default-schema", "kafka") + .put("kafka.nodes", "localhost:12345,localhost:23456") + .put("kafka.connect-timeout", "1h") + .put("kafka.buffer-size", "1MB") + .put("kafka.hide-internal-columns", "false") + .put("kafka.fetch-size", "10000000") + .put("kafka.zk-endpoint", "localhost:2181") + .build(); + + KafkaConnectorConfig expected = new KafkaConnectorConfig() + .setTableDescriptionDir(new File("/var/lib/kafka")) + .setTableNames("table1, table2, table3") + .setDefaultSchema("kafka") + .setNodes("localhost:12345, localhost:23456") + .setKafkaConnectTimeout("1h") + .setKafkaBufferSize("1MB") + .setHideInternalColumns(false) + .setFetchSize(10000000) + .setZkEndpoint("localhost:2181"); + + ConfigAssertions.assertFullMapping(properties, expected); + } +} diff --git a/presto-kafka07/src/test/java/com/facebook/presto/kafka/TestKafkaDistributed.java b/presto-kafka07/src/test/java/com/facebook/presto/kafka/TestKafkaDistributed.java new file mode 100644 index 0000000000000..c31e40f5d62e8 --- /dev/null +++ b/presto-kafka07/src/test/java/com/facebook/presto/kafka/TestKafkaDistributed.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.kafka.util.EmbeddedKafka; +import com.facebook.presto.tests.AbstractTestQueries; +import io.airlift.tpch.TpchTable; +import org.testng.annotations.AfterClass; +import org.testng.annotations.Test; + +import java.io.IOException; + +import static com.facebook.presto.kafka.KafkaQueryRunner.createKafkaQueryRunner; +import static com.facebook.presto.kafka.util.EmbeddedKafka.createEmbeddedKafka; + +@Test +public class TestKafkaDistributed + extends AbstractTestQueries +{ + private final EmbeddedKafka embeddedKafka; + + public TestKafkaDistributed() + throws Exception + { + this(createEmbeddedKafka()); + } + + public TestKafkaDistributed(EmbeddedKafka embeddedKafka) + throws Exception + { + super(() -> createKafkaQueryRunner(embeddedKafka, TpchTable.getTables())); + this.embeddedKafka = embeddedKafka; + } + + @AfterClass(alwaysRun = true) + public void destroy() + throws IOException + { + embeddedKafka.close(); + } +} diff --git a/presto-kafka07/src/test/java/com/facebook/presto/kafka/TestKafkaIntegrationSmokeTest.java b/presto-kafka07/src/test/java/com/facebook/presto/kafka/TestKafkaIntegrationSmokeTest.java new file mode 100644 index 0000000000000..faf02f0bfb80f --- /dev/null +++ b/presto-kafka07/src/test/java/com/facebook/presto/kafka/TestKafkaIntegrationSmokeTest.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.kafka.util.EmbeddedKafka; +import com.facebook.presto.tests.AbstractTestIntegrationSmokeTest; +import org.testng.annotations.AfterClass; +import org.testng.annotations.Test; + +import java.io.IOException; + +import static com.facebook.presto.kafka.KafkaQueryRunner.createKafkaQueryRunner; +import static com.facebook.presto.kafka.util.EmbeddedKafka.createEmbeddedKafka; +import static io.airlift.tpch.TpchTable.ORDERS; + +@Test +public class TestKafkaIntegrationSmokeTest + extends AbstractTestIntegrationSmokeTest +{ + private final EmbeddedKafka embeddedKafka; + + public TestKafkaIntegrationSmokeTest() + throws Exception + { + this(createEmbeddedKafka()); + } + + public TestKafkaIntegrationSmokeTest(EmbeddedKafka embeddedKafka) + throws Exception + { + super(() -> createKafkaQueryRunner(embeddedKafka, ORDERS)); + this.embeddedKafka = embeddedKafka; + } + + @AfterClass(alwaysRun = true) + public void destroy() + throws IOException + { + embeddedKafka.close(); + } +} diff --git a/presto-kafka07/src/test/java/com/facebook/presto/kafka/TestKafkaPlugin.java b/presto-kafka07/src/test/java/com/facebook/presto/kafka/TestKafkaPlugin.java new file mode 100644 index 0000000000000..ad5e19b0eef4f --- /dev/null +++ b/presto-kafka07/src/test/java/com/facebook/presto/kafka/TestKafkaPlugin.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.spi.connector.Connector; +import com.facebook.presto.spi.connector.ConnectorFactory; +import com.facebook.presto.testing.TestingConnectorContext; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import static com.google.common.collect.Iterables.getOnlyElement; +import static io.airlift.testing.Assertions.assertInstanceOf; +import static org.testng.Assert.assertNotNull; + +@Test +public class TestKafkaPlugin +{ + @Test + public void testSpinup() + { + KafkaPlugin plugin = new KafkaPlugin(); + + ConnectorFactory factory = getOnlyElement(plugin.getConnectorFactories()); + assertInstanceOf(factory, KafkaConnectorFactory.class); + + Connector c = factory.create( + "test-connector", + ImmutableMap.builder() + .put("kafka.table-names", "test") + .put("kafka.nodes", "localhost:9092") + .build(), + new TestingConnectorContext()); + assertNotNull(c); + } +} diff --git a/presto-kafka07/src/test/java/com/facebook/presto/kafka/TestManySegments.java b/presto-kafka07/src/test/java/com/facebook/presto/kafka/TestManySegments.java new file mode 100644 index 0000000000000..56c39553fe762 --- /dev/null +++ b/presto-kafka07/src/test/java/com/facebook/presto/kafka/TestManySegments.java @@ -0,0 +1,118 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.Session; +import com.facebook.presto.kafka.util.EmbeddedKafka; +import com.facebook.presto.kafka.util.TestUtils; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.type.BigintType; +import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.tests.StandaloneQueryRunner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import kafka.javaapi.producer.ProducerData; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.util.Collections; +import java.util.Properties; +import java.util.UUID; + +import static com.facebook.presto.kafka.util.EmbeddedKafka.CloseableProducer; +import static com.facebook.presto.kafka.util.TestUtils.createEmptyTopicDescription; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static com.facebook.presto.testing.assertions.Assert.assertEquals; + +@Test(singleThreaded = true) +public class TestManySegments +{ + private static final Session SESSION = testSessionBuilder() + .setCatalog("kafka") + .setSchema("default") + .build(); + + private EmbeddedKafka embeddedKafka; + private String topicName; + private StandaloneQueryRunner queryRunner; + + @BeforeClass + public void startKafka() + throws Exception + { + embeddedKafka = EmbeddedKafka.createEmbeddedKafka(); + embeddedKafka.start(); + + topicName = "test_" + UUID.randomUUID().toString().replaceAll("-", "_"); + + Properties topicProperties = new Properties(); + topicProperties.setProperty("segment.bytes", "1048576"); + + embeddedKafka.createTopics(1, 1, topicProperties, topicName); + + try (CloseableProducer producer = embeddedKafka.createProducer()) { + int jMax = 10_000; + int iMax = 100_000 / jMax; + for (long i = 0; i < iMax; i++) { + ImmutableList.Builder> builder = ImmutableList.builder(); + for (long j = 0; j < jMax; j++) { + builder.add(new ProducerData<>(topicName, i, Collections.singletonList(ImmutableMap.of("id", String.format("%05d", i * jMax + j), "value", UUID.randomUUID().toString())))); + } + producer.send(builder.build()); + } + } + } + + @AfterClass(alwaysRun = true) + public void stopKafka() + throws Exception + { + embeddedKafka.close(); + } + + @BeforeMethod + public void spinUp() + throws Exception + { + this.queryRunner = new StandaloneQueryRunner(SESSION); + + TestUtils.installKafkaPlugin(embeddedKafka, queryRunner, + ImmutableMap.builder() + .put(createEmptyTopicDescription(topicName, new SchemaTableName("default", topicName))) + .build()); + } + + @AfterMethod(alwaysRun = true) + public void tearDown() + throws Exception + { + queryRunner.close(); + } + + @Test + public void testManySegments() + throws Exception + { + MaterializedResult result = queryRunner.execute("SELECT count(_message) from " + topicName); + + MaterializedResult expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT) + .row(100000L) + .build(); + + assertEquals(result, expected); + } +} diff --git a/presto-kafka07/src/test/java/com/facebook/presto/kafka/TestMinimalFunctionality.java b/presto-kafka07/src/test/java/com/facebook/presto/kafka/TestMinimalFunctionality.java new file mode 100644 index 0000000000000..d26e77b4f9dcf --- /dev/null +++ b/presto-kafka07/src/test/java/com/facebook/presto/kafka/TestMinimalFunctionality.java @@ -0,0 +1,147 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka; + +import com.facebook.presto.Session; +import com.facebook.presto.kafka.util.EmbeddedKafka; +import com.facebook.presto.kafka.util.TestUtils; +import com.facebook.presto.metadata.QualifiedObjectName; +import com.facebook.presto.metadata.TableHandle; +import com.facebook.presto.security.AllowAllAccessControl; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.type.BigintType; +import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.tests.StandaloneQueryRunner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import kafka.javaapi.producer.ProducerData; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.util.Collections; +import java.util.Optional; +import java.util.Properties; +import java.util.UUID; + +import static com.facebook.presto.kafka.util.EmbeddedKafka.CloseableProducer; +import static com.facebook.presto.kafka.util.TestUtils.createEmptyTopicDescription; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static com.facebook.presto.testing.assertions.Assert.assertEquals; +import static com.facebook.presto.transaction.TransactionBuilder.transaction; +import static org.testng.Assert.assertTrue; + +@Test(singleThreaded = true) +public class TestMinimalFunctionality +{ + private static final Session SESSION = testSessionBuilder() + .setCatalog("kafka") + .setSchema("default") + .build(); + + private EmbeddedKafka embeddedKafka; + private String topicName; + private StandaloneQueryRunner queryRunner; + + @BeforeClass + public void startKafka() + throws Exception + { + embeddedKafka = EmbeddedKafka.createEmbeddedKafka(); + embeddedKafka.start(); + } + + @AfterClass + public void stopKafka() + throws Exception + { + embeddedKafka.close(); + } + + @BeforeMethod + public void spinUp() + throws Exception + { + this.topicName = "test_" + UUID.randomUUID().toString().replaceAll("-", "_"); + + Properties topicProperties = new Properties(); + embeddedKafka.createTopics(2, 1, topicProperties, topicName); + + this.queryRunner = new StandaloneQueryRunner(SESSION); + + TestUtils.installKafkaPlugin(embeddedKafka, queryRunner, + ImmutableMap.builder() + .put(createEmptyTopicDescription(topicName, new SchemaTableName("default", topicName))) + .build()); + } + + @AfterMethod + public void tearDown() + throws Exception + { + queryRunner.close(); + } + + private void createMessages(String topicName, int count) + { + try (CloseableProducer producer = embeddedKafka.createProducer()) { + ImmutableList.Builder> builder = ImmutableList.builder(); + for (long i = 0; i < count; i++) { + Object message = ImmutableMap.of("id", Long.toString(i), "value", UUID.randomUUID().toString()); + builder.add(new ProducerData<>(topicName, i, Collections.singletonList(message))); + } + producer.send(builder.build()); + } + } + + @Test + public void testTopicExists() + throws Exception + { + QualifiedObjectName name = new QualifiedObjectName("kafka", "default", topicName); + + transaction(queryRunner.getTransactionManager(), new AllowAllAccessControl()) + .singleStatement() + .execute(SESSION, session -> { + Optional handle = queryRunner.getServer().getMetadata().getTableHandle(session, name); + assertTrue(handle.isPresent()); + }); + } + + @Test + public void testTopicHasData() + throws Exception + { + MaterializedResult result = queryRunner.execute("SELECT count(1) from " + topicName); + + MaterializedResult expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT) + .row(0L) + .build(); + + assertEquals(result, expected); + + int count = 1000; + createMessages(topicName, count); + Thread.sleep(10_000); + result = queryRunner.execute("SELECT count(1) from " + topicName); + + expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT) + .row((long) count) + .build(); + + assertEquals(result, expected); + } +} diff --git a/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/CodecSupplier.java b/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/CodecSupplier.java new file mode 100644 index 0000000000000..05f7a035869f3 --- /dev/null +++ b/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/CodecSupplier.java @@ -0,0 +1,71 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka.util; + +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.type.Type; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer; +import com.google.common.collect.ImmutableMap; +import io.airlift.json.JsonCodec; +import io.airlift.json.JsonCodecFactory; +import io.airlift.json.ObjectMapperProvider; + +import java.util.function.Supplier; + +import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; + +public final class CodecSupplier + implements Supplier> +{ + private final Metadata metadata; + private final JsonCodecFactory codecFactory; + private final Class clazz; + + public CodecSupplier(Class clazz, Metadata metadata) + { + this.clazz = clazz; + this.metadata = metadata; + ObjectMapperProvider objectMapperProvider = new ObjectMapperProvider(); + objectMapperProvider.setJsonDeserializers(ImmutableMap.of(Type.class, new TypeDeserializer())); + this.codecFactory = new JsonCodecFactory(objectMapperProvider); + } + + @Override + public JsonCodec get() + { + return codecFactory.jsonCodec(clazz); + } + + private class TypeDeserializer + extends FromStringDeserializer + { + private static final long serialVersionUID = 1L; + + public TypeDeserializer() + { + super(Type.class); + } + + @Override + protected Type _deserialize(String value, DeserializationContext context) + { + Type type = metadata.getType(parseTypeSignature(value)); + if (type == null) { + throw new IllegalArgumentException(String.valueOf("Unknown type " + value)); + } + return type; + } + } +} diff --git a/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/EmbeddedKafka.java b/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/EmbeddedKafka.java new file mode 100644 index 0000000000000..2ea3a57f3b48a --- /dev/null +++ b/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/EmbeddedKafka.java @@ -0,0 +1,166 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka.util; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import com.google.common.io.Files; +import com.google.common.io.MoreFiles; +import kafka.javaapi.producer.Producer; +import kafka.producer.ProducerConfig; +import kafka.server.KafkaConfig; +import kafka.server.KafkaServerStartable; +import scala.Option; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicBoolean; + +import static com.facebook.presto.kafka.util.TestUtils.findUnusedPort; +import static com.facebook.presto.kafka.util.TestUtils.toProperties; +import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; +import static java.util.Objects.requireNonNull; + +public class EmbeddedKafka + implements Closeable +{ + private final EmbeddedZookeeper zookeeper; + private final int port; + private final File kafkaDataDir; + private final KafkaServerStartable kafka; + + private final AtomicBoolean started = new AtomicBoolean(); + private final AtomicBoolean stopped = new AtomicBoolean(); + + public static EmbeddedKafka createEmbeddedKafka() + throws IOException + { + return new EmbeddedKafka(new EmbeddedZookeeper(), new Properties()); + } + + public static EmbeddedKafka createEmbeddedKafka(Properties overrideProperties) + throws IOException + { + return new EmbeddedKafka(new EmbeddedZookeeper(), overrideProperties); + } + + EmbeddedKafka(EmbeddedZookeeper zookeeper, Properties overrideProperties) + throws IOException + { + this.zookeeper = requireNonNull(zookeeper, "zookeeper is null"); + requireNonNull(overrideProperties, "overrideProperties is null"); + + this.port = findUnusedPort(); + this.kafkaDataDir = Files.createTempDir(); + + Map properties = ImmutableMap.builder() + .put("brokerid", "0") + .put("regionid", "1") + .put("hostname", "localhost") + .put("num.partitions", "2") + .put("log.flush.interval", "10000") + .put("log.default.flush.interval.ms", "1000") + .put("log.retention.hours", "1") + .put("log.segment.bytes", "10248576") + .put("log.file.size", "1048576") + .put("auto.create.topics.enable", "false") + .put("zookeeper.connection.timeout.ms", "1000000") + .put("port", Integer.toString(port)) + .put("log.dir", kafkaDataDir.getAbsolutePath()) + .put("zookeeper.connect", zookeeper.getConnectString()) + .put("zk.connect", zookeeper.getConnectString()) + .putAll(Maps.fromProperties(overrideProperties)) + .build(); + + KafkaConfig config = new KafkaConfig(toProperties(properties)); + this.kafka = new KafkaServerStartable(config, Option.empty()); + } + + public void start() + throws InterruptedException, IOException + { + if (!started.getAndSet(true)) { + zookeeper.start(); + kafka.startup(); + } + } + + @Override + public void close() + throws IOException + { + if (started.get() && !stopped.getAndSet(true)) { + kafka.shutdown(); + kafka.awaitShutdown(); + zookeeper.close(); + MoreFiles.deleteRecursively(kafkaDataDir.toPath(), ALLOW_INSECURE); + } + } + + public void createTopics(String... topics) + { + createTopics(2, 1, new Properties(), topics); + } + + public void createTopics(int partitions, int replication, Properties topicProperties, String... topics) + { + // noop + } + + public CloseableProducer createProducer() + { + Map properties = ImmutableMap.builder() + .put("broker.list", String.format("0:%s", getConnectString())) + .put("serializer.class", JsonEncoder.class.getName()) + .put("key.serializer.class", NumberEncoder.class.getName()) + .put("request.required.acks", "1") + .build(); + + ProducerConfig producerConfig = new ProducerConfig(toProperties(properties)); + return new CloseableProducer<>(producerConfig); + } + + public static class CloseableProducer + extends Producer + implements AutoCloseable + { + public CloseableProducer(ProducerConfig config) + { + super(config); + } + } + + public int getZookeeperPort() + { + return zookeeper.getPort(); + } + + public int getPort() + { + return port; + } + + public String getConnectString() + { + return "localhost:" + Integer.toString(port); + } + + public String getZookeeperConnectString() + { + return zookeeper.getConnectString(); + } +} diff --git a/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/EmbeddedZookeeper.java b/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/EmbeddedZookeeper.java new file mode 100644 index 0000000000000..d312179421fd8 --- /dev/null +++ b/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/EmbeddedZookeeper.java @@ -0,0 +1,101 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka.util; + +import com.google.common.io.Files; +import com.google.common.io.MoreFiles; +import org.apache.zookeeper.server.NIOServerCnxnFactory; +import org.apache.zookeeper.server.ServerCnxnFactory; +import org.apache.zookeeper.server.ZooKeeperServer; +import org.apache.zookeeper.server.persistence.FileTxnSnapLog; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.concurrent.atomic.AtomicBoolean; + +import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; + +public class EmbeddedZookeeper + implements Closeable +{ + private final int port; + private final File zkDataDir; + private final ZooKeeperServer zkServer; + private final ServerCnxnFactory cnxnFactory; + + private final AtomicBoolean started = new AtomicBoolean(); + private final AtomicBoolean stopped = new AtomicBoolean(); + + public EmbeddedZookeeper() + throws IOException + { + this(TestUtils.findUnusedPort()); + } + + public EmbeddedZookeeper(int port) + throws IOException + { + this.port = port; + zkDataDir = Files.createTempDir(); + zkServer = new ZooKeeperServer(); + + FileTxnSnapLog ftxn = new FileTxnSnapLog(zkDataDir, zkDataDir); + zkServer.setTxnLogFactory(ftxn); + zkServer.setMinSessionTimeout(60000); + zkServer.setMaxSessionTimeout(120000); + + cnxnFactory = NIOServerCnxnFactory.createFactory(new InetSocketAddress(port), 300); + } + + public void start() + throws InterruptedException, IOException + { + if (!started.getAndSet(true)) { + cnxnFactory.startup(zkServer); + } + } + + @Override + public void close() + throws IOException + { + if (started.get() && !stopped.getAndSet(true)) { + cnxnFactory.shutdown(); + try { + cnxnFactory.join(); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + if (zkServer.isRunning()) { + zkServer.shutdown(); + } + + MoreFiles.deleteRecursively(zkDataDir.toPath(), ALLOW_INSECURE); + } + } + + public String getConnectString() + { + return "127.0.0.1:" + Integer.toString(port); + } + + public int getPort() + { + return port; + } +} diff --git a/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/JsonEncoder.java b/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/JsonEncoder.java new file mode 100644 index 0000000000000..65e54985c894d --- /dev/null +++ b/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/JsonEncoder.java @@ -0,0 +1,39 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.kafka.util; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Throwables; +import kafka.message.Message; +import kafka.serializer.Encoder; + +import java.io.IOException; + +public class JsonEncoder + implements Encoder +{ + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Override + public Message toMessage(Object o) + { + try { + return new Message(objectMapper.writeValueAsBytes(o)); + } + catch (IOException e) { + throw Throwables.propagate(e); + } + } +} diff --git a/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/KafkaLoader.java b/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/KafkaLoader.java new file mode 100644 index 0000000000000..f13fff45d3053 --- /dev/null +++ b/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/KafkaLoader.java @@ -0,0 +1,158 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka.util; + +import com.facebook.presto.Session; +import com.facebook.presto.client.Column; +import com.facebook.presto.client.QueryData; +import com.facebook.presto.client.QueryStatusInfo; +import com.facebook.presto.server.testing.TestingPrestoServer; +import com.facebook.presto.spi.type.TimeZoneKey; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.Varchars; +import com.facebook.presto.tests.AbstractTestingPrestoClient; +import com.facebook.presto.tests.ResultsSession; +import com.google.common.collect.ImmutableMap; +import kafka.javaapi.producer.Producer; +import kafka.javaapi.producer.ProducerData; +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.ISODateTimeFormat; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.DateTimeEncoding.unpackMillisUtc; +import static com.facebook.presto.spi.type.DateType.DATE; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.TimeType.TIME; +import static com.facebook.presto.spi.type.TimeWithTimeZoneType.TIME_WITH_TIME_ZONE; +import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; +import static com.facebook.presto.spi.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE; +import static com.facebook.presto.util.DateTimeUtils.parseTimeLiteral; +import static com.facebook.presto.util.DateTimeUtils.parseTimestampWithTimeZone; +import static com.facebook.presto.util.DateTimeUtils.parseTimestampWithoutTimeZone; +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + +public class KafkaLoader + extends AbstractTestingPrestoClient +{ + private static final DateTimeFormatter ISO8601_FORMATTER = ISODateTimeFormat.dateTime(); + + private final String topicName; + private final Producer producer; + private final AtomicLong count = new AtomicLong(); + + public KafkaLoader(Producer producer, + String topicName, + TestingPrestoServer prestoServer, + Session defaultSession) + { + super(prestoServer, defaultSession); + + this.topicName = topicName; + this.producer = producer; + } + + @Override + public ResultsSession getResultSession(Session session) + { + requireNonNull(session, "session is null"); + return new KafkaLoadingSession(session); + } + + private class KafkaLoadingSession + implements ResultsSession + { + private final AtomicReference> types = new AtomicReference<>(); + + private final TimeZoneKey timeZoneKey; + + private KafkaLoadingSession(Session session) + { + this.timeZoneKey = session.getTimeZoneKey(); + } + + @Override + public void addResults(QueryStatusInfo statusInfo, QueryData data) + { + if (types.get() == null && statusInfo.getColumns() != null) { + types.set(getTypes(statusInfo.getColumns())); + } + + if (data.getData() != null) { + checkState(types.get() != null, "Data without types received!"); + List columns = statusInfo.getColumns(); + for (List fields : data.getData()) { + ImmutableMap.Builder builder = ImmutableMap.builder(); + for (int i = 0; i < fields.size(); i++) { + Type type = types.get().get(i); + Object value = convertValue(fields.get(i), type); + if (value != null) { + builder.put(columns.get(i).getName(), value); + } + } + + producer.send(new ProducerData<>(topicName, count.getAndIncrement(), Collections.singletonList(builder.build()))); + } + } + } + + @Override + public Void build(Map setSessionProperties, Set resetSessionProperties) + { + return null; + } + + private Object convertValue(Object value, Type type) + { + if (value == null) { + return null; + } + + if (BOOLEAN.equals(type) || Varchars.isVarcharType(type)) { + return value; + } + if (BIGINT.equals(type)) { + return ((Number) value).longValue(); + } + if (INTEGER.equals(type)) { + return ((Number) value).intValue(); + } + if (DOUBLE.equals(type)) { + return ((Number) value).doubleValue(); + } + if (DATE.equals(type)) { + return value; + } + if (TIME.equals(type)) { + return ISO8601_FORMATTER.print(parseTimeLiteral(timeZoneKey, (String) value)); + } + if (TIMESTAMP.equals(type)) { + return ISO8601_FORMATTER.print(parseTimestampWithoutTimeZone(timeZoneKey, (String) value)); + } + if (TIME_WITH_TIME_ZONE.equals(type) || TIMESTAMP_WITH_TIME_ZONE.equals(type)) { + return ISO8601_FORMATTER.print(unpackMillisUtc(parseTimestampWithTimeZone(timeZoneKey, (String) value))); + } + throw new AssertionError("unhandled type: " + type); + } + } +} diff --git a/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/NumberEncoder.java b/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/NumberEncoder.java new file mode 100644 index 0000000000000..07b627b4cff9b --- /dev/null +++ b/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/NumberEncoder.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka.util; + +import kafka.message.Message; +import kafka.serializer.Encoder; + +import java.nio.ByteBuffer; + +public class NumberEncoder + implements Encoder +{ + @Override + public Message toMessage(Number value) + { + ByteBuffer buf = ByteBuffer.allocate(8); + buf.putLong(value == null ? 0L : value.longValue()); + return new Message(buf.array()); + } +} diff --git a/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/NumberPartitioner.java b/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/NumberPartitioner.java new file mode 100644 index 0000000000000..a1225b3588136 --- /dev/null +++ b/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/NumberPartitioner.java @@ -0,0 +1,33 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka.util; + +import kafka.cluster.Partition; +import kafka.producer.Partitioner; +import scala.collection.Seq; + +import static java.lang.Math.toIntExact; + +public class NumberPartitioner + implements Partitioner +{ + @Override + public int partition(Object key, Seq partitions) + { + if (key instanceof Number) { + return toIntExact(((Number) key).longValue() % partitions.size()); + } + return 0; + } +} diff --git a/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/TestUtils.java b/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/TestUtils.java new file mode 100644 index 0000000000000..5e194df60de6b --- /dev/null +++ b/presto-kafka07/src/test/java/com/facebook/presto/kafka/util/TestUtils.java @@ -0,0 +1,96 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kafka.util; + +import com.facebook.presto.kafka.KafkaPlugin; +import com.facebook.presto.kafka.KafkaTopicDescription; +import com.facebook.presto.metadata.QualifiedObjectName; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.TestingPrestoClient; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableMap; +import com.google.common.io.ByteStreams; +import io.airlift.json.JsonCodec; + +import java.io.IOException; +import java.net.ServerSocket; +import java.util.AbstractMap; +import java.util.Map; +import java.util.Properties; + +import static com.facebook.presto.kafka.util.EmbeddedKafka.CloseableProducer; +import static java.lang.String.format; + +public final class TestUtils +{ + private TestUtils() {} + + public static int findUnusedPort() + throws IOException + { + try (ServerSocket socket = new ServerSocket(0)) { + return socket.getLocalPort(); + } + } + + public static Properties toProperties(Map map) + { + Properties properties = new Properties(); + for (Map.Entry entry : map.entrySet()) { + properties.setProperty(entry.getKey(), entry.getValue()); + } + return properties; + } + + public static void installKafkaPlugin(EmbeddedKafka embeddedKafka, QueryRunner queryRunner, Map topicDescriptions) + { + KafkaPlugin kafkaPlugin = new KafkaPlugin(); + kafkaPlugin.setTableDescriptionSupplier(() -> topicDescriptions); + queryRunner.installPlugin(kafkaPlugin); + + Map kafkaConfig = ImmutableMap.of( + "kafka.nodes", embeddedKafka.getConnectString(), + "kafka.table-names", Joiner.on(",").join(topicDescriptions.keySet()), + "kafka.connect-timeout", "120s", + "kafka.default-schema", "default", + "kafka.zk-endpoint", embeddedKafka.getZookeeperConnectString()); + queryRunner.createCatalog("kafka", "kafka07", kafkaConfig); + } + + public static void loadTpchTopic(EmbeddedKafka embeddedKafka, TestingPrestoClient prestoClient, String topicName, QualifiedObjectName tpchTableName) + { + try (CloseableProducer producer = embeddedKafka.createProducer(); + KafkaLoader tpchLoader = new KafkaLoader(producer, topicName, prestoClient.getServer(), prestoClient.getDefaultSession())) { + tpchLoader.execute(format("SELECT * from %s", tpchTableName)); + } + } + + public static Map.Entry loadTpchTopicDescription(JsonCodec topicDescriptionJsonCodec, String topicName, SchemaTableName schemaTableName) + throws IOException + { + KafkaTopicDescription tpchTemplate = topicDescriptionJsonCodec.fromJson(ByteStreams.toByteArray(TestUtils.class.getResourceAsStream(format("/tpch/%s.json", schemaTableName.getTableName())))); + + return new AbstractMap.SimpleImmutableEntry<>( + schemaTableName, + new KafkaTopicDescription(schemaTableName.getTableName(), schemaTableName.getSchemaName(), topicName, tpchTemplate.getKey(), tpchTemplate.getMessage())); + } + + public static Map.Entry createEmptyTopicDescription(String topicName, SchemaTableName schemaTableName) + { + return new AbstractMap.SimpleImmutableEntry<>( + schemaTableName, + new KafkaTopicDescription(schemaTableName.getTableName(), schemaTableName.getSchemaName(), topicName, null, null)); + } +} diff --git a/presto-kafka07/src/test/resources/tpch/customer.json b/presto-kafka07/src/test/resources/tpch/customer.json new file mode 100644 index 0000000000000..cf800fc0a9bb9 --- /dev/null +++ b/presto-kafka07/src/test/resources/tpch/customer.json @@ -0,0 +1,61 @@ +{ + "tableName": "customer", + "schemaName": "tpch", + "topicName": "tpch.customer", + "key": { + "dataFormat": "raw", + "fields": [ + { + "name": "kafka_key", + "dataFormat": "LONG", + "type": "BIGINT", + "hidden": "true" + } + ] + }, + "message": { + "dataFormat": "json", + "fields": [ + { + "name": "custkey", + "mapping": "custkey", + "type": "BIGINT" + }, + { + "name": "name", + "mapping": "name", + "type": "VARCHAR(25)" + }, + { + "name": "address", + "mapping": "address", + "type": "VARCHAR(40)" + }, + { + "name": "nationkey", + "mapping": "nationkey", + "type": "BIGINT" + }, + { + "name": "phone", + "mapping": "phone", + "type": "VARCHAR(15)" + }, + { + "name": "acctbal", + "mapping": "acctbal", + "type": "DOUBLE" + }, + { + "name": "mktsegment", + "mapping": "mktsegment", + "type": "VARCHAR(10)" + }, + { + "name": "comment", + "mapping": "comment", + "type": "VARCHAR(117)" + } + ] + } +} diff --git a/presto-kafka07/src/test/resources/tpch/lineitem.json b/presto-kafka07/src/test/resources/tpch/lineitem.json new file mode 100644 index 0000000000000..2250fddf77714 --- /dev/null +++ b/presto-kafka07/src/test/resources/tpch/lineitem.json @@ -0,0 +1,104 @@ +{ + "tableName": "lineitem", + "schemaName": "tpch", + "topicName": "tpch.lineitem", + "key": { + "dataFormat": "raw", + "fields": [ + { + "name": "kafka_key", + "dataFormat": "LONG", + "type": "BIGINT", + "hidden": "true" + } + ] + }, + "message": { + "dataFormat": "json", + "fields": [ + { + "name": "orderkey", + "mapping": "orderkey", + "type": "BIGINT" + }, + { + "name": "partkey", + "mapping": "partkey", + "type": "BIGINT" + }, + { + "name": "suppkey", + "mapping": "suppkey", + "type": "BIGINT" + }, + { + "name": "linenumber", + "mapping": "linenumber", + "type": "INTEGER" + }, + { + "name": "quantity", + "mapping": "quantity", + "type": "DOUBLE" + }, + { + "name": "extendedprice", + "mapping": "extendedprice", + "type": "DOUBLE" + }, + { + "name": "discount", + "mapping": "discount", + "type": "DOUBLE" + }, + { + "name": "tax", + "mapping": "tax", + "type": "DOUBLE" + }, + { + "name": "returnflag", + "mapping": "returnflag", + "type": "VARCHAR(1)" + }, + { + "name": "linestatus", + "mapping": "linestatus", + "type": "VARCHAR(1)" + }, + { + "name": "shipdate", + "mapping": "shipdate", + "type": "DATE", + "dataFormat": "iso8601" + }, + { + "name": "commitdate", + "mapping": "commitdate", + "type": "DATE", + "dataFormat": "iso8601" + }, + { + "name": "receiptdate", + "mapping": "receiptdate", + "type": "DATE", + "dataFormat": "iso8601" + }, + { + "name": "shipinstruct", + "mapping": "shipinstruct", + "type": "VARCHAR(25)" + }, + { + "name": "shipmode", + "mapping": "shipmode", + "type": "VARCHAR(7)" + }, + { + "name": "comment", + "mapping": "comment", + "type": "VARCHAR(44)" + } + ] + } +} diff --git a/presto-kafka07/src/test/resources/tpch/nation.json b/presto-kafka07/src/test/resources/tpch/nation.json new file mode 100644 index 0000000000000..0749a1249d76e --- /dev/null +++ b/presto-kafka07/src/test/resources/tpch/nation.json @@ -0,0 +1,41 @@ +{ + "tableName": "nation", + "schemaName": "tpch", + "topicName": "tpch.nation", + "key": { + "dataFormat": "raw", + "fields": [ + { + "name": "kafka_key", + "dataFormat": "LONG", + "type": "BIGINT", + "hidden": "true" + } + ] + }, + "message": { + "dataFormat": "json", + "fields": [ + { + "name": "nationkey", + "mapping": "nationkey", + "type": "BIGINT" + }, + { + "name": "name", + "mapping": "name", + "type": "VARCHAR(25)" + }, + { + "name": "regionkey", + "mapping": "regionkey", + "type": "BIGINT" + }, + { + "name": "comment", + "mapping": "comment", + "type": "VARCHAR(152)" + } + ] + } +} diff --git a/presto-kafka07/src/test/resources/tpch/orders.json b/presto-kafka07/src/test/resources/tpch/orders.json new file mode 100644 index 0000000000000..f5eb594083a4e --- /dev/null +++ b/presto-kafka07/src/test/resources/tpch/orders.json @@ -0,0 +1,67 @@ +{ + "tableName": "orders", + "schemaName": "tpch", + "topicName": "tpch.orders", + "key": { + "dataFormat": "raw", + "fields": [ + { + "name": "kafka_key", + "dataFormat": "LONG", + "type": "BIGINT", + "hidden": "true" + } + ] + }, + "message": { + "dataFormat": "json", + "fields": [ + { + "name": "orderkey", + "mapping": "orderkey", + "type": "BIGINT" + }, + { + "name": "custkey", + "mapping": "custkey", + "type": "BIGINT" + }, + { + "name": "orderstatus", + "mapping": "orderstatus", + "type": "VARCHAR(1)" + }, + { + "name": "totalprice", + "mapping": "totalprice", + "type": "DOUBLE" + }, + { + "name": "orderdate", + "mapping": "orderdate", + "type": "DATE", + "dataFormat": "iso8601" + }, + { + "name": "orderpriority", + "mapping": "orderpriority", + "type": "VARCHAR(15)" + }, + { + "name": "clerk", + "mapping": "clerk", + "type": "VARCHAR(15)" + }, + { + "name": "shippriority", + "mapping": "shippriority", + "type": "INTEGER" + }, + { + "name": "comment", + "mapping": "comment", + "type": "VARCHAR(79)" + } + ] + } +} diff --git a/presto-kafka07/src/test/resources/tpch/part.json b/presto-kafka07/src/test/resources/tpch/part.json new file mode 100644 index 0000000000000..3c56b27955f1d --- /dev/null +++ b/presto-kafka07/src/test/resources/tpch/part.json @@ -0,0 +1,66 @@ +{ + "tableName": "part", + "schemaName": "tpch", + "topicName": "tpch.part", + "key": { + "dataFormat": "raw", + "fields": [ + { + "name": "kafka_key", + "dataFormat": "LONG", + "type": "BIGINT", + "hidden": "true" + } + ] + }, + "message": { + "dataFormat": "json", + "fields": [ + { + "name": "partkey", + "mapping": "partkey", + "type": "BIGINT" + }, + { + "name": "name", + "mapping": "name", + "type": "VARCHAR(55)" + }, + { + "name": "mfgr", + "mapping": "mfgr", + "type": "VARCHAR(25)" + }, + { + "name": "brand", + "mapping": "brand", + "type": "VARCHAR(10)" + }, + { + "name": "type", + "mapping": "type", + "type": "VARCHAR(25)" + }, + { + "name": "size", + "mapping": "size", + "type": "INTEGER" + }, + { + "name": "container", + "mapping": "container", + "type": "VARCHAR(10)" + }, + { + "name": "retailprice", + "mapping": "retailprice", + "type": "DOUBLE" + }, + { + "name": "comment", + "mapping": "comment", + "type": "VARCHAR(23)" + } + ] + } +} diff --git a/presto-kafka07/src/test/resources/tpch/partsupp.json b/presto-kafka07/src/test/resources/tpch/partsupp.json new file mode 100644 index 0000000000000..cd6d6364f87f7 --- /dev/null +++ b/presto-kafka07/src/test/resources/tpch/partsupp.json @@ -0,0 +1,46 @@ +{ + "tableName": "partsupp", + "schemaName": "tpch", + "topicName": "tpch.partsupp", + "key": { + "dataFormat": "raw", + "fields": [ + { + "name": "kafka_key", + "dataFormat": "LONG", + "type": "BIGINT", + "hidden": "true" + } + ] + }, + "message": { + "dataFormat": "json", + "fields": [ + { + "name": "partkey", + "mapping": "partkey", + "type": "BIGINT" + }, + { + "name": "suppkey", + "mapping": "suppkey", + "type": "BIGINT" + }, + { + "name": "availqty", + "mapping": "availqty", + "type": "INTEGER" + }, + { + "name": "supplycost", + "mapping": "supplycost", + "type": "DOUBLE" + }, + { + "name": "comment", + "mapping": "comment", + "type": "VARCHAR(199)" + } + ] + } +} diff --git a/presto-kafka07/src/test/resources/tpch/region.json b/presto-kafka07/src/test/resources/tpch/region.json new file mode 100644 index 0000000000000..4aca016e22cca --- /dev/null +++ b/presto-kafka07/src/test/resources/tpch/region.json @@ -0,0 +1,36 @@ +{ + "tableName": "region", + "schemaName": "tpch", + "topicName": "tpch.region", + "key": { + "dataFormat": "raw", + "fields": [ + { + "name": "kafka_key", + "dataFormat": "LONG", + "type": "BIGINT", + "hidden": "true" + } + ] + }, + "message": { + "dataFormat": "json", + "fields": [ + { + "name": "regionkey", + "mapping": "regionkey", + "type": "BIGINT" + }, + { + "name": "name", + "mapping": "name", + "type": "VARCHAR(25)" + }, + { + "name": "comment", + "mapping": "comment", + "type": "VARCHAR(152)" + } + ] + } +} diff --git a/presto-kafka07/src/test/resources/tpch/supplier.json b/presto-kafka07/src/test/resources/tpch/supplier.json new file mode 100644 index 0000000000000..af9d24c86a833 --- /dev/null +++ b/presto-kafka07/src/test/resources/tpch/supplier.json @@ -0,0 +1,56 @@ +{ + "tableName": "supplier", + "schemaName": "tpch", + "topicName": "tpch.supplier", + "key": { + "dataFormat": "raw", + "fields": [ + { + "name": "kafka_key", + "dataFormat": "LONG", + "type": "BIGINT", + "hidden": "true" + } + ] + }, + "message": { + "dataFormat": "json", + "fields": [ + { + "name": "suppkey", + "mapping": "suppkey", + "type": "BIGINT" + }, + { + "name": "name", + "mapping": "name", + "type": "VARCHAR(25)" + }, + { + "name": "address", + "mapping": "address", + "type": "VARCHAR(40)" + }, + { + "name": "nationkey", + "mapping": "nationkey", + "type": "BIGINT" + }, + { + "name": "phone", + "mapping": "phone", + "type": "VARCHAR(15)" + }, + { + "name": "acctbal", + "mapping": "acctbal", + "type": "DOUBLE" + }, + { + "name": "comment", + "mapping": "comment", + "type": "VARCHAR(101)" + } + ] + } +} diff --git a/presto-local-file/pom.xml b/presto-local-file/pom.xml index 600213db280c2..1671dfe836f51 100644 --- a/presto-local-file/pom.xml +++ b/presto-local-file/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-local-file diff --git a/presto-main/etc/config.properties b/presto-main/etc/config.properties index a84531ecb4e82..d275748712fab 100644 --- a/presto-main/etc/config.properties +++ b/presto-main/etc/config.properties @@ -34,6 +34,7 @@ plugin.bundles=\ ../presto-hive-hadoop2/pom.xml,\ ../presto-example-http/pom.xml,\ ../presto-kafka/pom.xml, \ + ../presto-kafka07/pom.xml, \ ../presto-tpch/pom.xml, \ ../presto-local-file/pom.xml, \ ../presto-mysql/pom.xml,\ diff --git a/presto-main/pom.xml b/presto-main/pom.xml index e4d71c29b141c..c885bd0491c2f 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-main diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java index ca55deb9f1d64..42ac7b711d612 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/DiscoveryNodeManager.java @@ -133,12 +133,18 @@ public void startPollingNodeStates() Set deadNodes = difference(nodeStates.keySet(), aliveNodeIds).immutableCopy(); nodeStates.keySet().removeAll(deadNodes); + if (deadNodes.size() > 0) { + log.warn("Dead nodes: %s", deadNodes); + } + // Add new nodes for (Node node : aliveNodes) { nodeStates.putIfAbsent(node.getNodeIdentifier(), new RemoteNodeState(httpClient, uriBuilderFrom(node.getHttpUri()).appendPath("/v1/info/state").build())); } + log.debug("Number of alive nodes: %d", nodeStates.size()); + // Schedule refresh nodeStates.values().forEach(RemoteNodeState::asyncRefresh); }, 1, 5, TimeUnit.SECONDS); diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index bd5916c4b323f..013e6750c3dec 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -107,8 +107,6 @@ public void run() new ServerMainModule(sqlParserOptions), new GracefulShutdownModule()); - modules.addAll(getAdditionalModules()); - Bootstrap app = new Bootstrap(modules.build()); try { @@ -141,11 +139,6 @@ public void run() } } - protected Iterable getAdditionalModules() - { - return ImmutableList.of(); - } - private static void updateConnectorIds(Announcer announcer, CatalogManager metadata, ServerConfig serverConfig, NodeSchedulerConfig schedulerConfig) { // get existing announcement diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoSystemRequirements.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoSystemRequirements.java index 001c420453f0c..8d6b604d249a6 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoSystemRequirements.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoSystemRequirements.java @@ -54,7 +54,7 @@ private static void verifyJvmVendor() { String vendor = StandardSystemProperty.JAVA_VENDOR.value(); if (!"Oracle Corporation".equals(vendor)) { - failRequirement("Presto requires an Oracle or OpenJDK JVM (found %s)", vendor); + warnRequirement("Presto requires an Oracle or OpenJDK JVM (found %s)", vendor); } } diff --git a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java index 9318249e8e77d..37d6b797050b6 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java @@ -147,6 +147,7 @@ import com.facebook.presto.sql.planner.CompilerConfig; import com.facebook.presto.sql.planner.LocalExecutionPlanner; import com.facebook.presto.sql.planner.NodePartitioningManager; +import com.facebook.presto.sql.planner.Symbol; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.FunctionCall; import com.facebook.presto.transaction.ForTransactionManager; @@ -442,6 +443,8 @@ protected void setup(Binder binder) jsonBinder(binder).addSerializerBinding(Expression.class).to(ExpressionSerializer.class); jsonBinder(binder).addDeserializerBinding(Expression.class).to(ExpressionDeserializer.class); jsonBinder(binder).addDeserializerBinding(FunctionCall.class).to(FunctionCallDeserializer.class); + jsonBinder(binder).addKeySerializerBinding(Symbol.class).to(Symbol.SymbolKeySerializer.class); + jsonBinder(binder).addKeyDeserializerBinding(Symbol.class).to(Symbol.SymbolKeyDeserializer.class); // query monitor configBinder(binder).bindConfig(QueryMonitorConfig.class); diff --git a/presto-main/src/main/java/com/facebook/presto/server/protocol/StatementResource.java b/presto-main/src/main/java/com/facebook/presto/server/protocol/StatementResource.java index da49095deb1db..3a10eb3f3fc8b 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/protocol/StatementResource.java +++ b/presto-main/src/main/java/com/facebook/presto/server/protocol/StatementResource.java @@ -70,6 +70,7 @@ import static com.facebook.presto.client.PrestoHeaders.PRESTO_STARTED_TRANSACTION_ID; import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.google.common.base.Strings.isNullOrEmpty; +import static com.google.common.base.Strings.nullToEmpty; import static com.google.common.net.HttpHeaders.X_FORWARDED_PROTO; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static io.airlift.concurrent.Threads.threadsNamed; @@ -139,6 +140,23 @@ public Response createQuery( proto = uriInfo.getRequestUri().getScheme(); } + // The Teradata Presto ODBC Driver checks node version to decide the Presto's statement + // protocol and sends test queries about PREPARE statement. + // Rewrite the statement so that Presto always returns version for the compatible protocol. + // Ban the statement which will never be consumed by the driver. + if (nullToEmpty(servletRequest.getHeader("User-Agent")).equals("Teradata Presto ODBC Driver")) { + if (statement.equals("select node_version from system.runtime.nodes where coordinator=true")) { + statement = "select '0.148' as node_version"; + } + else if (statement.equals("DESCRIBE OUTPUT prepare_test_stmt")) { + throw new WebApplicationException(Response + .status(Status.BAD_REQUEST) + .type(MediaType.TEXT_PLAIN) + .entity("SQL statement is known, and wouldn't be consumed by this driver") + .build()); + } + } + SessionContext sessionContext = new HttpRequestSessionContext(servletRequest); ExchangeClient exchangeClient = exchangeClientSupplier.get(new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), StatementResource.class.getSimpleName())); diff --git a/presto-main/src/main/java/com/facebook/presto/server/security/AuthenticationFilter.java b/presto-main/src/main/java/com/facebook/presto/server/security/AuthenticationFilter.java index d7b6c1a4a64fb..fcb691b46acee 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/security/AuthenticationFilter.java +++ b/presto-main/src/main/java/com/facebook/presto/server/security/AuthenticationFilter.java @@ -44,11 +44,13 @@ public class AuthenticationFilter implements Filter { private final List authenticators; + private final String httpAuthenticationPathRegex; @Inject - public AuthenticationFilter(List authenticators) + public AuthenticationFilter(Set authenticators, SecurityConfig securityConfig) { this.authenticators = ImmutableList.copyOf(authenticators); + this.httpAuthenticationPathRegex = requireNonNull(securityConfig.getHttpAuthenticationPathRegex(), "httpAuthenticationPathRegex is null"); } @Override @@ -64,8 +66,8 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo HttpServletRequest request = (HttpServletRequest) servletRequest; HttpServletResponse response = (HttpServletResponse) servletResponse; - // skip authentication if non-secure or not configured - if (!request.isSecure() || authenticators.isEmpty()) { + // skip authentication if (not configured) or (non-secure and not match httpAuthenticationPathRegex) + if (authenticators.isEmpty() || (!request.isSecure() && !request.getPathInfo().matches(httpAuthenticationPathRegex))) { nextFilter.doFilter(request, response); return; } diff --git a/presto-main/src/main/java/com/facebook/presto/server/security/KerberosAuthenticator.java b/presto-main/src/main/java/com/facebook/presto/server/security/KerberosAuthenticator.java index 6ecd91a6f1dd4..581b86ff6ac2c 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/security/KerberosAuthenticator.java +++ b/presto-main/src/main/java/com/facebook/presto/server/security/KerberosAuthenticator.java @@ -64,8 +64,9 @@ public KerberosAuthenticator(KerberosConfig config) System.setProperty("java.security.krb5.conf", config.getKerberosConfig().getAbsolutePath()); try { + boolean isCompleteServicePrinciple = config.getServiceName().contains("@"); String hostname = InetAddress.getLocalHost().getCanonicalHostName().toLowerCase(Locale.US); - String servicePrincipal = config.getServiceName() + "/" + hostname; + String servicePrincipal = isCompleteServicePrinciple ? config.getServiceName() : config.getServiceName() + "/" + hostname; loginContext = new LoginContext("", null, null, new Configuration() { @Override @@ -91,7 +92,7 @@ public AppConfigurationEntry[] getAppConfigurationEntry(String name) loginContext.login(); serverCredential = doAs(loginContext.getSubject(), () -> gssManager.createCredential( - gssManager.createName(config.getServiceName() + "@" + hostname, GSSName.NT_HOSTBASED_SERVICE), + isCompleteServicePrinciple ? gssManager.createName(config.getServiceName(), GSSName.NT_USER_NAME) : gssManager.createName(config.getServiceName() + "@" + hostname, GSSName.NT_HOSTBASED_SERVICE), INDEFINITE_LIFETIME, new Oid[] { new Oid("1.2.840.113554.1.2.2"), // kerberos 5 diff --git a/presto-main/src/main/java/com/facebook/presto/server/security/SecurityConfig.java b/presto-main/src/main/java/com/facebook/presto/server/security/SecurityConfig.java index 68293cfcba462..ec92f0dbccf79 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/security/SecurityConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/server/security/SecurityConfig.java @@ -33,6 +33,8 @@ public class SecurityConfig private List authenticationTypes = ImmutableList.of(); + private String httpAuthenticationPathRegex = "^\b$"; + public enum AuthenticationType { CERTIFICATE, @@ -67,4 +69,18 @@ public SecurityConfig setAuthenticationTypes(String types) .collect(toImmutableList()); return this; } + + @NotNull + public String getHttpAuthenticationPathRegex() + { + return httpAuthenticationPathRegex; + } + + @Config("http-server.http.authentication.path.regex") + @ConfigDescription("Regex of path that needs to be authenticated for non-secured http request") + public SecurityConfig setHttpAuthenticationPathRegex(String regex) + { + httpAuthenticationPathRegex = regex; + return this; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/server/security/ServerSecurityModule.java b/presto-main/src/main/java/com/facebook/presto/server/security/ServerSecurityModule.java index 81cfb07628828..d65e0900a9660 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/security/ServerSecurityModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/security/ServerSecurityModule.java @@ -42,6 +42,7 @@ protected void setup(Binder binder) { newSetBinder(binder, Filter.class, TheServlet.class).addBinding() .to(AuthenticationFilter.class).in(Scopes.SINGLETON); + configBinder(binder).bindConfig(SecurityConfig.class); binder.bind(PasswordAuthenticatorManager.class).in(Scopes.SINGLETON); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java index 6d4c555b1afc4..7a8229eca45a9 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java @@ -377,8 +377,8 @@ protected Type visitSymbolReference(SymbolReference node, StackableAstVisitorCon return setExpressionType(node, resolvedField.get().getType()); } } - Type type = symbolTypes.get(Symbol.from(node)); - return setExpressionType(node, type); + + return setExpressionType(node, getType(node)); } @Override @@ -1337,6 +1337,16 @@ else if (typeOnlyCoercions.contains(ref)) { typeOnlyCoercions.remove(ref); } } + + private Type getType(SymbolReference symbolReference) + { + Type type = symbolTypes.get(Symbol.from(symbolReference)); + if (type == null) { + type = symbolTypes.get(Symbol.withEmptyFields(symbolReference)); + } + + return type; + } } private static class Context diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java index 4912deb7362e8..bbaab4738858e 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java @@ -230,6 +230,7 @@ import static com.facebook.presto.sql.analyzer.ExpressionAnalyzer.getExpressionTypes; import static com.facebook.presto.sql.analyzer.ExpressionAnalyzer.getExpressionTypesFromInput; import static com.facebook.presto.sql.planner.ExpressionNodeInliner.replaceExpression; +import static com.facebook.presto.sql.planner.Symbol.withEmptyFields; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.COORDINATOR_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_ARBITRARY_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_BROADCAST_DISTRIBUTION; @@ -577,6 +578,16 @@ public TypeProvider getTypes() return types; } + public Type getType(Symbol symbol) + { + Type type = types.get(symbol); + if (type == null) { + type = types.get(withEmptyFields(symbol)); + } + + return type; + } + public Optional getIndexSourceContext() { return indexSourceContext; @@ -2758,7 +2769,7 @@ private static List toTypes(Map layout, LocalExecutionPla return range(0, channelCount) .mapToObj(channelLayout::get) - .map(context.getTypes()::get) + .map(context::getType) .collect(toImmutableList()); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LookupSymbolResolver.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LookupSymbolResolver.java index dcd0ef41cd0cd..474f6f4496b76 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LookupSymbolResolver.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LookupSymbolResolver.java @@ -18,13 +18,16 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; +import java.util.function.Function; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static java.util.Objects.requireNonNull; public class LookupSymbolResolver implements SymbolResolver { + private final Map assignmentSymbolLookup; private final Map assignments; private final Map bindings; @@ -33,6 +36,8 @@ public LookupSymbolResolver(Map assignments, Map assignments, Map typeMapper(Set dependencies) + { + Map typeMap = types.allTypes(); + ImmutableMap.Builder builder = ImmutableMap.builder(); + for (Symbol symbol : dependencies) { + if (typeMap.containsKey(symbol)) { + builder.put(symbol, types.get(symbol)); + continue; + } + Symbol alternativeSymbol = withEmptyFields(symbol); + if (typeMap.containsKey(alternativeSymbol)) { + builder.put(symbol, types.get(alternativeSymbol)); + } + } + return builder.build(); + } + @Override public PlanNode visitOutput(OutputNode node, RewriteContext context) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java index aca93237dd1fb..4ad3b9d1143cc 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java @@ -44,11 +44,13 @@ import com.facebook.presto.sql.planner.iterative.rule.MergeLimitWithTopN; import com.facebook.presto.sql.planner.iterative.rule.MergeLimits; import com.facebook.presto.sql.planner.iterative.rule.MultipleDistinctAggregationToMarkDistinct; +import com.facebook.presto.sql.planner.iterative.rule.PickColumnHandles; import com.facebook.presto.sql.planner.iterative.rule.PickTableLayout; import com.facebook.presto.sql.planner.iterative.rule.PruneAggregationColumns; import com.facebook.presto.sql.planner.iterative.rule.PruneAggregationSourceColumns; import com.facebook.presto.sql.planner.iterative.rule.PruneCountAggregationOverScalar; import com.facebook.presto.sql.planner.iterative.rule.PruneCrossJoinColumns; +import com.facebook.presto.sql.planner.iterative.rule.PruneFilterColumnFields; import com.facebook.presto.sql.planner.iterative.rule.PruneFilterColumns; import com.facebook.presto.sql.planner.iterative.rule.PruneIndexSourceColumns; import com.facebook.presto.sql.planner.iterative.rule.PruneJoinChildrenColumns; @@ -57,9 +59,11 @@ import com.facebook.presto.sql.planner.iterative.rule.PruneMarkDistinctColumns; import com.facebook.presto.sql.planner.iterative.rule.PruneOrderByInAggregation; import com.facebook.presto.sql.planner.iterative.rule.PruneOutputColumns; +import com.facebook.presto.sql.planner.iterative.rule.PruneProjectColumnFields; import com.facebook.presto.sql.planner.iterative.rule.PruneProjectColumns; import com.facebook.presto.sql.planner.iterative.rule.PruneSemiJoinColumns; import com.facebook.presto.sql.planner.iterative.rule.PruneSemiJoinFilteringSourceColumns; +import com.facebook.presto.sql.planner.iterative.rule.PruneTableScanColumnFields; import com.facebook.presto.sql.planner.iterative.rule.PruneTableScanColumns; import com.facebook.presto.sql.planner.iterative.rule.PruneTopNColumns; import com.facebook.presto.sql.planner.iterative.rule.PruneValuesColumns; @@ -205,6 +209,11 @@ public PlanOptimizers( new PruneLimitColumns(), new PruneTableScanColumns()); + Set> fieldPruningRules = ImmutableSet.of( + new PruneProjectColumnFields(), + new PruneFilterColumnFields(), + new PruneTableScanColumnFields()); + IterativeOptimizer inlineProjections = new IterativeOptimizer( ruleStats, statsCalculator, @@ -454,6 +463,25 @@ public PlanOptimizers( .add(new InlineProjections()) .build())); + // Optimize the nested fields. + builder.add(new IterativeOptimizer( + ruleStats, + statsCalculator, + costCalculator, + ImmutableSet.>builder() + .addAll(fieldPruningRules) + .add(new RemoveRedundantIdentityProjections()) + .build())); + + // Update the table layouts after pruning. + builder.add(new IterativeOptimizer( + ruleStats, + statsCalculator, + costCalculator, + ImmutableSet.>builder() + .add(new PickColumnHandles(metadata)) + .build())); + // Optimizers above this don't understand local exchanges, so be careful moving this. builder.add(new AddLocalExchanges(metadata, sqlParser)); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/Symbol.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/Symbol.java index 8168d4b4ef6f8..8a396e1db9901 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/Symbol.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/Symbol.java @@ -13,11 +13,25 @@ */ package com.facebook.presto.sql.planner; +import com.facebook.presto.sql.tree.DereferenceExpression; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.SymbolReference; import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonValue; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.KeyDeserializer; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.google.common.collect.ImmutableSet; +import io.airlift.json.ObjectMapperProvider; +import java.io.IOException; +import java.util.Objects; +import java.util.Set; + +import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; @@ -25,35 +39,94 @@ public class Symbol implements Comparable { private final String name; + private final Set fields; public static Symbol from(Expression expression) { checkArgument(expression instanceof SymbolReference, "Unexpected expression: %s", expression); - return new Symbol(((SymbolReference) expression).getName()); + return new Symbol(((SymbolReference) expression).getName(), ((SymbolReference) expression).getFields()); + } + + public static Symbol withField(DereferenceExpression expression) + { + checkArgument(expression.getBase() instanceof SymbolReference, "Unexpected base expression: %s", expression.getBase()); + return new Symbol(((SymbolReference) expression.getBase()).getName(), ImmutableSet.of(expression.getField().getValue())); + } + + public static Symbol withAllFields(SymbolReference expression) + { + return new Symbol(expression.getName(), expression.getFields()); + } + + public static Symbol withEmptyFields(Symbol symbol) + { + return new Symbol(symbol.getName()); + } + + public static Symbol withEmptyFields(SymbolReference symbolReference) + { + return new Symbol(symbolReference.getName()); + } + + public static Symbol merge(Symbol a, Symbol b) + { + checkArgument(a.getName().equalsIgnoreCase(b.getName()), "Symbols with different names cannot be merged: %s vs %s", a.getName(), b.getName()); + return new Symbol(a.getName(), mergeAndPruneFields(a.fields, b.fields)); + } + + public static boolean contains(Symbol a, Symbol b) + { + if (!a.getName().equalsIgnoreCase(b.getName())) { + return false; + } + + if (!a.getFields().isEmpty() && !a.getFields().containsAll(b.getFields())) { + return false; + } + + return true; } - @JsonCreator public Symbol(String name) + { + this(name, ImmutableSet.of()); + } + + @JsonCreator + public Symbol( + @JsonProperty("name") String name, + @JsonProperty("fields") Set fields) { requireNonNull(name, "name is null"); + requireNonNull(fields, "fields is null"); this.name = name; + this.fields = fields; } - @JsonValue + @JsonProperty public String getName() { return name; } + @JsonProperty + public Set getFields() + { + return fields; + } + public SymbolReference toSymbolReference() { - return new SymbolReference(name); + return new SymbolReference(name, fields); } @Override public String toString() { - return name; + return toStringHelper(this) + .add("name", name) + .add("fields", fields) + .toString(); } @Override @@ -72,13 +145,17 @@ public boolean equals(Object o) return false; } + if (!fields.equals(symbol.fields)) { + return false; + } + return true; } @Override public int hashCode() { - return name.hashCode(); + return Objects.hash(name, fields); } @Override @@ -86,4 +163,41 @@ public int compareTo(Symbol o) { return name.compareTo(o.name); } + + private static Set mergeAndPruneFields(Set a, Set b) + { + if (a.size() == 0 || b.size() == 0) { + return ImmutableSet.of(); + } + return ImmutableSet.builder() + .addAll(a) + .addAll(b) + .build(); + } + + public static class SymbolKeySerializer + extends JsonSerializer + { + private static final ObjectMapper MAPPER = new ObjectMapperProvider().get(); + + @Override + public void serialize(Symbol symbol, JsonGenerator jsonGenerator, SerializerProvider serializers) + throws IOException + { + jsonGenerator.writeFieldName(MAPPER.writeValueAsString(symbol)); + } + } + + public static class SymbolKeyDeserializer + extends KeyDeserializer + { + private static final ObjectMapper MAPPER = new ObjectMapperProvider().get(); + + @Override + public Object deserializeKey(String key, DeserializationContext ctxt) + throws IOException + { + return MAPPER.readValue(key, Symbol.class); + } + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolToInputRewriter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolToInputRewriter.java index e965af2a88164..8e5fc10d8e35b 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolToInputRewriter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolToInputRewriter.java @@ -23,17 +23,22 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; +import java.util.function.Function; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static java.util.Objects.requireNonNull; public class SymbolToInputRewriter { private final Map symbolToChannelMapping; + private final Map nameToSymbolMapping; public SymbolToInputRewriter(Map symbolToChannelMapping) { requireNonNull(symbolToChannelMapping, "symbolToChannelMapping is null"); this.symbolToChannelMapping = ImmutableMap.copyOf(symbolToChannelMapping); + this.nameToSymbolMapping = symbolToChannelMapping.keySet().stream() + .collect(toImmutableMap(Symbol::getName, Function.identity())); } public Expression rewrite(Expression expression) @@ -44,6 +49,9 @@ public Expression rewrite(Expression expression) public Expression rewriteSymbolReference(SymbolReference node, Context context, ExpressionTreeRewriter treeRewriter) { Integer channel = symbolToChannelMapping.get(Symbol.from(node)); + if (channel == null) { + channel = symbolToChannelMapping.get(nameToSymbolMapping.get(node.getName())); + } if (channel == null) { Preconditions.checkArgument(context.isInLambda(), "Cannot resolve symbol %s", node.getName()); return node; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolsExtractor.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolsExtractor.java index c04347ee11b8d..7e146ea99342b 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolsExtractor.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/SymbolsExtractor.java @@ -24,9 +24,12 @@ import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.SymbolReference; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import static com.facebook.presto.sql.planner.ExpressionExtractor.extractExpressions; @@ -107,6 +110,31 @@ public static Set extractOutputSymbols(PlanNode planNode, Lookup lookup) .collect(toImmutableSet()); } + public static Map extractUniqueWithFields(Iterable expressions) + { + Map symbolMap = new HashMap<>(); + for (Expression expression : expressions) { + extractAllWithFields(expression).forEach(symbol -> { + String name = symbol.getName(); + if (symbolMap.containsKey(name)) { + symbolMap.put(name, Symbol.merge(symbolMap.get(name), symbol)); + } + else { + symbolMap.put(name, symbol); + } + }); + } + + return ImmutableMap.copyOf(symbolMap); + } + + public static List extractAllWithFields(Expression expression) + { + ImmutableList.Builder builder = ImmutableList.builder(); + new SymbolWithFieldsBuilderVisitor().process(expression, builder); + return builder.build(); + } + private static class SymbolBuilderVisitor extends DefaultExpressionTraversalVisitor> { @@ -147,4 +175,27 @@ protected Void visitIdentifier(Identifier node, ImmutableSet.Builder> + { + @Override + protected Void visitDereferenceExpression(DereferenceExpression expression, ImmutableList.Builder builder) + { + if (expression.getBase() instanceof SymbolReference) { + builder.add(Symbol.withField(expression)); + } + else { + process(expression.getBase(), builder); + } + return null; + } + + @Override + protected Void visitSymbolReference(SymbolReference node, ImmutableList.Builder builder) + { + builder.add(Symbol.withAllFields(node)); + return null; + } + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/TypeProvider.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/TypeProvider.java index 1bd17351e80c6..bae92b0c9b746 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/TypeProvider.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/TypeProvider.java @@ -18,6 +18,7 @@ import java.util.Collections; import java.util.Map; +import java.util.Optional; import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; @@ -51,6 +52,15 @@ public Type get(Symbol symbol) requireNonNull(symbol, "symbol is null"); Type type = types.get(symbol); + if (type == null) { + Optional alternative = types.entrySet().stream() + .filter(e -> e.getKey().getName().equals(symbol.getName())) + .findAny() + .map(Map.Entry::getValue); + if (alternative.isPresent()) { + type = alternative.get(); + } + } checkArgument(type != null, "no type found found for symbol '%s'", symbol); return type; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PickColumnHandles.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PickColumnHandles.java new file mode 100644 index 0000000000000..80929e44ce76b --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PickColumnHandles.java @@ -0,0 +1,102 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.metadata.TableLayoutResult; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.Constraint; +import com.facebook.presto.spi.predicate.FieldSet; +import com.facebook.presto.sql.planner.Symbol; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.facebook.presto.sql.planner.plan.TableScanNode; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.sql.planner.plan.Patterns.tableScan; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static java.util.Objects.requireNonNull; + +public class PickColumnHandles + implements Rule +{ + private final Metadata metadata; + private static final Pattern PATTERN = tableScan(); + + public PickColumnHandles(Metadata metadata) + { + this.metadata = requireNonNull(metadata, "metadata is null"); + } + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public Result apply(TableScanNode node, Captures captures, Context context) + { + List pruningColumns = node.getAssignments().keySet().stream() + .filter(symbol -> !symbol.getFields().isEmpty()) + .collect(toImmutableList()); + + if (pruningColumns.isEmpty()) { + return Result.empty(); + } + + List> fieldSets = pruningColumns.stream() + .map(symbol -> new FieldSet<>(node.getAssignments().get(symbol), symbol.getFields())) + .collect(toImmutableList()); + + List layouts = metadata.getLayouts( + context.getSession(), + node.getTable(), + new Constraint<>(Optional.of(fieldSets)), + Optional.of(ImmutableSet.copyOf(node.getAssignments().values()))); + + TableLayoutResult layout = layouts.get(0); + + if (!layout.getLayout().getColumns().isPresent() || layout.getLayout().getColumns().get().isEmpty()) { + return Result.empty(); + } + + List columns = layout.getLayout().getColumns().get(); + + Map assignments = node.getAssignments().entrySet().stream() + .map(entry -> { + int index = pruningColumns.indexOf(entry.getKey()); + if (index >= 0) { + return Maps.immutableEntry(entry.getKey(), columns.get(index)); + } + return entry; + }).collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); + + return Result.ofPlanNode(new TableScanNode( + node.getId(), + node.getTable(), + node.getOutputSymbols(), + assignments, + node.getLayout(), + node.getCurrentConstraint(), + node.getOriginalConstraint())); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ProjectOffPushDownFieldRule.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ProjectOffPushDownFieldRule.java new file mode 100644 index 0000000000000..8c8b76527c9cc --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ProjectOffPushDownFieldRule.java @@ -0,0 +1,66 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.matching.Capture; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.sql.planner.PlanNodeIdAllocator; +import com.facebook.presto.sql.planner.Symbol; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.facebook.presto.sql.planner.plan.PlanNode; +import com.facebook.presto.sql.planner.plan.ProjectNode; +import com.google.common.collect.ImmutableList; + +import java.util.Optional; +import java.util.Set; + +import static com.facebook.presto.matching.Capture.newCapture; +import static com.facebook.presto.sql.planner.iterative.rule.Util.pruneNestedFields; +import static com.facebook.presto.sql.planner.plan.Patterns.project; +import static com.facebook.presto.sql.planner.plan.Patterns.source; + +public abstract class ProjectOffPushDownFieldRule + implements Rule +{ + private final Capture targetCapture = newCapture(); + + private final Pattern targetPattern; + + protected ProjectOffPushDownFieldRule(Pattern targetPattern) + { + this.targetPattern = targetPattern; + } + + @Override + public Pattern getPattern() + { + return project() + .with(source().matching(targetPattern.capturedAs(targetCapture))); + } + + @Override + public Result apply(ProjectNode parent, Captures captures, Context context) + { + N targetNode = captures.get(targetCapture); + + return pruneNestedFields(targetNode.getOutputSymbols(), parent.getAssignments().getExpressions()) + .flatMap(prunedOutputs -> this.pushDownProjectOff(context.getIdAllocator(), targetNode, prunedOutputs)) + .map(newChild -> parent.replaceChildren(ImmutableList.of(newChild))) + .map(Result::ofPlanNode) + .orElse(Result.empty()); + } + + protected abstract Optional pushDownProjectOff(PlanNodeIdAllocator idAllocator, N targetNode, Set referencedOutputs); +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneFilterColumnFields.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneFilterColumnFields.java new file mode 100644 index 0000000000000..b2ec89703975a --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneFilterColumnFields.java @@ -0,0 +1,57 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.sql.planner.PlanNodeIdAllocator; +import com.facebook.presto.sql.planner.Symbol; +import com.facebook.presto.sql.planner.plan.FilterNode; +import com.facebook.presto.sql.planner.plan.PlanNode; +import com.google.common.collect.ImmutableSet; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.facebook.presto.sql.planner.SymbolsExtractor.extractUniqueWithFields; +import static com.facebook.presto.sql.planner.iterative.rule.Util.restrictChildOutputs; +import static com.facebook.presto.sql.planner.plan.Patterns.filter; + +public class PruneFilterColumnFields + extends ProjectOffPushDownFieldRule +{ + public PruneFilterColumnFields() + { + super(filter()); + } + + @Override + protected Optional pushDownProjectOff(PlanNodeIdAllocator idAllocator, FilterNode filterNode, Set referencedOutputs) + { + Map filterInputsMap = new HashMap<>(extractUniqueWithFields(ImmutableSet.of(filterNode.getPredicate()))); + + referencedOutputs.forEach(symbol -> { + String name = symbol.getName(); + if (filterInputsMap.containsKey(name)) { + filterInputsMap.put(name, Symbol.merge(filterInputsMap.get(name), symbol)); + } + else { + filterInputsMap.put(name, symbol); + } + }); + Set prunedFilterInputs = ImmutableSet.copyOf(filterInputsMap.values()); + + return restrictChildOutputs(idAllocator, filterNode, prunedFilterInputs); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneProjectColumnFields.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneProjectColumnFields.java new file mode 100644 index 0000000000000..adb31cbaf94ed --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneProjectColumnFields.java @@ -0,0 +1,62 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.sql.planner.PlanNodeIdAllocator; +import com.facebook.presto.sql.planner.Symbol; +import com.facebook.presto.sql.planner.plan.PlanNode; +import com.facebook.presto.sql.planner.plan.ProjectNode; +import com.facebook.presto.sql.tree.Expression; +import com.facebook.presto.sql.tree.SymbolReference; +import com.google.common.collect.Maps; + +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; + +import static com.facebook.presto.sql.planner.plan.Patterns.project; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static java.util.Objects.requireNonNull; + +public class PruneProjectColumnFields + extends ProjectOffPushDownFieldRule +{ + public PruneProjectColumnFields() + { + super(project()); + } + + @Override + protected Optional pushDownProjectOff( + PlanNodeIdAllocator idAllocator, + ProjectNode childProjectNode, + Set referencedOutputs) + { + Map symbolMap = referencedOutputs.stream().collect(toImmutableMap(Symbol::getName, Function.identity())); + Function, Map.Entry> rewrite = entry -> { + Symbol newSymbol = requireNonNull(symbolMap.get(entry.getKey().getName())); + if (entry.getValue() instanceof SymbolReference) { + return Maps.immutableEntry(newSymbol, (Expression) newSymbol.toSymbolReference()); + } + return Maps.immutableEntry(newSymbol, entry.getValue()); + }; + + return Optional.of( + new ProjectNode( + childProjectNode.getId(), + childProjectNode.getSource(), + childProjectNode.getAssignments().filter(symbol -> symbolMap.containsKey(symbol.getName())).rewriteEntry(rewrite))); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneTableScanColumnFields.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneTableScanColumnFields.java new file mode 100644 index 0000000000000..639c0862cfc11 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PruneTableScanColumnFields.java @@ -0,0 +1,70 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.sql.planner.PlanNodeIdAllocator; +import com.facebook.presto.sql.planner.Symbol; +import com.facebook.presto.sql.planner.plan.PlanNode; +import com.facebook.presto.sql.planner.plan.TableScanNode; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.facebook.presto.sql.planner.plan.Patterns.tableScan; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; + +public class PruneTableScanColumnFields + extends ProjectOffPushDownFieldRule +{ + public PruneTableScanColumnFields() + { + super(tableScan()); + } + + @Override + protected Optional pushDownProjectOff(PlanNodeIdAllocator idAllocator, TableScanNode tableScanNode, Set referencedOutputs) + { + Map referencedOutputsMap = referencedOutputs.stream() + .collect(toImmutableMap(Symbol::getName, symbol -> symbol)); + return Optional.of( + new TableScanNode( + tableScanNode.getId(), + tableScanNode.getTable(), + filteredCopy(tableScanNode.getOutputSymbols(), referencedOutputsMap), + filterKeys(tableScanNode.getAssignments(), referencedOutputsMap), + tableScanNode.getLayout(), + tableScanNode.getCurrentConstraint(), + tableScanNode.getOriginalConstraint())); + } + + private List filteredCopy(List outputSymbols, Map referencedOutputsMap) + { + return outputSymbols.stream() + .map(Symbol::getName) + .filter(referencedOutputsMap::containsKey) + .map(referencedOutputsMap::get) + .collect(toImmutableList()); + } + + private Map filterKeys(Map assignments, Map referencedOutputsMap) + { + return assignments.keySet().stream() + .filter(symbol -> referencedOutputsMap.containsKey(symbol.getName())) + .collect(toImmutableMap(symbol -> referencedOutputsMap.get(symbol.getName()), assignments::get)); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/Util.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/Util.java index 240d81dd73cad..31e422f9bf049 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/Util.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/Util.java @@ -25,12 +25,15 @@ import com.google.common.collect.Sets; import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableSet.toImmutableSet; class Util { @@ -55,6 +58,26 @@ public static Optional> pruneInputs(Collection availableInpu return Optional.of(prunedInputs); } + public static Optional> pruneNestedFields(Collection availableInputs, Collection expressions) + { + Set availableInputsSet = ImmutableSet.copyOf(availableInputs); + Map expressionInputsMap = SymbolsExtractor.extractUniqueWithFields(expressions); + + Set prunedInputs = availableInputsSet.stream() + .filter(symbol -> expressionInputsMap.containsKey(symbol.getName())) + .collect(toImmutableSet()); + + if (prunedInputs.stream().allMatch(symbol -> expressionInputsMap.get(symbol.getName()).getFields().equals(symbol.getFields()))) { + return Optional.empty(); + } + + Set fieldPrunedInputs = prunedInputs.stream() + .map(symbol -> expressionInputsMap.getOrDefault(symbol.getName(), symbol)) + .collect(toImmutableSet()); + + return Optional.of(fieldPrunedInputs); + } + /** * Transforms a plan like P->C->X to C->P->X */ @@ -70,14 +93,20 @@ public static PlanNode transpose(PlanNode parent, PlanNode child) */ public static Optional restrictOutputs(PlanNodeIdAllocator idAllocator, PlanNode node, Set permittedOutputs) { - List restrictedOutputs = node.getOutputSymbols().stream() - .filter(permittedOutputs::contains) - .collect(toImmutableList()); + Map permittedOutputsMap = new HashMap<>(); + permittedOutputs.forEach(symbol -> permittedOutputsMap.put(symbol.getName(), symbol)); - if (restrictedOutputs.size() == node.getOutputSymbols().size()) { + if (node.getOutputSymbols().stream().allMatch(symbol -> + permittedOutputsMap.containsKey(symbol.getName()) + && permittedOutputsMap.get(symbol.getName()).getFields().equals(symbol.getFields()))) { return Optional.empty(); } + List restrictedOutputs = node.getOutputSymbols().stream() + .filter(symbol -> permittedOutputsMap.containsKey(symbol.getName())) + .map(symbol -> permittedOutputsMap.get(symbol.getName())) + .collect(toImmutableList()); + return Optional.of( new ProjectNode( idAllocator.getNextId(), diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Assignments.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Assignments.java index 8b6eeafa18123..e2647e674f1ca 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Assignments.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Assignments.java @@ -20,7 +20,6 @@ import com.facebook.presto.sql.tree.SymbolReference; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; @@ -32,8 +31,10 @@ import java.util.Map.Entry; import java.util.Set; import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collector; +import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkState; import static java.util.Arrays.asList; import static java.util.Objects.requireNonNull; @@ -108,6 +109,13 @@ public Assignments rewrite(Function rewrite) .collect(toAssignments()); } + public Assignments rewriteEntry(Function, Entry> rewrite) + { + return assignments.entrySet().stream() + .map(rewrite::apply) + .collect(toAssignments()); + } + public Assignments filter(Collection symbols) { return filter(symbols::contains); @@ -116,7 +124,7 @@ public Assignments filter(Collection symbols) public Assignments filter(Predicate predicate) { return assignments.entrySet().stream() - .filter(entry -> predicate.apply(entry.getKey())) + .filter(entry -> predicate.test(entry.getKey())) .collect(toAssignments()); } @@ -136,7 +144,7 @@ private Collector, Builder, Assignments> toAssignments left.putAll(right.build()); return left; }, - Assignments.Builder::build); + Builder::build); } public Collection getExpressions() @@ -190,6 +198,14 @@ public int hashCode() return assignments.hashCode(); } + @Override + public String toString() + { + return toStringHelper(this) + .add("assignments", assignments) + .toString(); + } + public static class Builder { private final Map assignments = new LinkedHashMap<>(); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java index ffd484a997cc0..97242258183d4 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java @@ -124,6 +124,7 @@ import static com.facebook.presto.execution.StageInfo.getAllStages; import static com.facebook.presto.operator.PipelineExecutionStrategy.UNGROUPED_EXECUTION; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static com.facebook.presto.sql.planner.Symbol.withEmptyFields; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION; import static com.facebook.presto.sql.planner.planPrinter.PlanNodeStatsSummarizer.aggregatePlanNodeStats; import static com.google.common.base.CaseFormat.UPPER_UNDERSCORE; @@ -1365,7 +1366,17 @@ private void printAssignments(Assignments assignments, int indent) private String formatOutputs(Iterable symbols) { - return Joiner.on(", ").join(Iterables.transform(symbols, input -> input + ":" + types.get(input).getDisplayName())); + return Joiner.on(", ").join(Iterables.transform(symbols, input -> input + ":" + getType(input).getDisplayName())); + } + + private Type getType(Symbol symbol) + { + Type type = types.get(symbol); + if (type == null) { + type = types.get(withEmptyFields(symbol)); + } + + return type; } private void printConstraint(int indent, ColumnHandle column, TupleDomain constraint) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/TypeValidator.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/TypeValidator.java index a077d9a4e4ae0..b9587ab2daf7d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/TypeValidator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/TypeValidator.java @@ -39,6 +39,8 @@ import java.util.Map; import static com.facebook.presto.sql.analyzer.ExpressionAnalyzer.getExpressionTypes; +import static com.facebook.presto.sql.planner.Symbol.from; +import static com.facebook.presto.sql.planner.Symbol.withEmptyFields; import static com.facebook.presto.type.UnknownType.UNKNOWN; import static com.google.common.base.Preconditions.checkArgument; import static java.util.Collections.emptyList; @@ -110,10 +112,11 @@ public Void visitProject(ProjectNode node, Void context) visitPlan(node, context); for (Map.Entry entry : node.getAssignments().entrySet()) { - Type expectedType = types.get(entry.getKey()); + Type expectedType = getType(entry.getKey()); if (entry.getValue() instanceof SymbolReference) { SymbolReference symbolReference = (SymbolReference) entry.getValue(); - verifyTypeSignature(entry.getKey(), expectedType.getTypeSignature(), types.get(Symbol.from(symbolReference)).getTypeSignature()); + Type actualType = getType(from(symbolReference)); + verifyTypeSignature(entry.getKey(), expectedType.getTypeSignature(), actualType.getTypeSignature()); continue; } Map, Type> expressionTypes = getExpressionTypes(session, metadata, sqlParser, types, entry.getValue(), emptyList() /* parameters already replaced */); @@ -132,9 +135,9 @@ public Void visitUnion(UnionNode node, Void context) ListMultimap symbolMapping = node.getSymbolMapping(); for (Symbol keySymbol : symbolMapping.keySet()) { List valueSymbols = symbolMapping.get(keySymbol); - Type expectedType = types.get(keySymbol); + Type expectedType = getType(keySymbol); for (Symbol valueSymbol : valueSymbols) { - verifyTypeSignature(keySymbol, expectedType.getTypeSignature(), types.get(valueSymbol).getTypeSignature()); + verifyTypeSignature(keySymbol, expectedType.getTypeSignature(), getType(valueSymbol).getTypeSignature()); } } @@ -154,14 +157,14 @@ private void checkWindowFunctions(Map functions) private void checkSignature(Symbol symbol, Signature signature) { - TypeSignature expectedTypeSignature = types.get(symbol).getTypeSignature(); + TypeSignature expectedTypeSignature = getType(symbol).getTypeSignature(); TypeSignature actualTypeSignature = signature.getReturnType(); verifyTypeSignature(symbol, expectedTypeSignature, actualTypeSignature); } private void checkCall(Symbol symbol, FunctionCall call) { - Type expectedType = types.get(symbol); + Type expectedType = getType(symbol); Map, Type> expressionTypes = getExpressionTypes(session, metadata, sqlParser, types, call, emptyList() /*parameters already replaced */); Type actualType = expressionTypes.get(NodeRef.of(call)); verifyTypeSignature(symbol, expectedType.getTypeSignature(), actualType.getTypeSignature()); @@ -189,5 +192,15 @@ private void verifyTypeSignature(Symbol symbol, TypeSignature expected, TypeSign checkArgument(expected.equals(actual), "type of symbol '%s' is expected to be %s, but the actual type is %s", symbol, expected, actual); } } + + private Type getType(Symbol symbol) + { + Type type = types.get(symbol); + if (type == null) { + type = types.get(withEmptyFields(symbol)); + } + + return type; + } } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java index 995158c31a792..b745517162a0d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java @@ -68,10 +68,12 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import static com.facebook.presto.sql.planner.optimizations.IndexJoinOptimizer.IndexKeyTracer; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; /** @@ -224,8 +226,8 @@ public Void visitFilter(FilterNode node, Set boundSymbols) Set inputs = createInputs(source, boundSymbols); checkDependencies(inputs, node.getOutputSymbols(), "Invalid node. Output symbols (%s) not in source plan output (%s)", node.getOutputSymbols(), node.getSource().getOutputSymbols()); - Set dependencies = SymbolsExtractor.extractUnique(node.getPredicate()); - checkDependencies(inputs, dependencies, "Invalid node. Predicate dependencies (%s) not in source plan output (%s)", dependencies, node.getSource().getOutputSymbols()); + Collection dependencies = SymbolsExtractor.extractUniqueWithFields(ImmutableSet.of(node.getPredicate())).values(); + checkDependenciesWithFields(inputs, dependencies, "Invalid node. Predicate dependencies (%s) not in source plan output (%s)", dependencies, node.getSource().getOutputSymbols()); return null; } @@ -246,10 +248,8 @@ public Void visitProject(ProjectNode node, Set boundSymbols) source.accept(this, boundSymbols); // visit child Set inputs = createInputs(source, boundSymbols); - for (Expression expression : node.getAssignments().getExpressions()) { - Set dependencies = SymbolsExtractor.extractUnique(expression); - checkDependencies(inputs, dependencies, "Invalid node. Expression dependencies (%s) not in source plan output (%s)", dependencies, inputs); - } + Collection dependencies = SymbolsExtractor.extractUniqueWithFields(node.getAssignments().getExpressions()).values(); + checkDependenciesWithFields(inputs, dependencies, "Invalid node. Expression dependencies (%s) not in source plan output (%s)", dependencies, inputs); return null; } @@ -622,4 +622,12 @@ private static void checkDependencies(Collection inputs, Collection inputs, Collection required, String message, Object... parameters) + { + Map inputsMap = inputs.stream() + .collect(toImmutableMap(Symbol::getName, Function.identity())); + checkArgument(required.stream() + .allMatch(symbol -> inputsMap.containsKey(symbol.getName()) && Symbol.contains(inputsMap.get(symbol.getName()), symbol)), message, parameters); + } } diff --git a/presto-main/src/main/java/com/facebook/presto/type/RowParametricType.java b/presto-main/src/main/java/com/facebook/presto/type/RowParametricType.java index 92892f8fa6115..22ab9bf8ce917 100644 --- a/presto-main/src/main/java/com/facebook/presto/type/RowParametricType.java +++ b/presto-main/src/main/java/com/facebook/presto/type/RowParametricType.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.type; +import com.facebook.presto.spi.type.NamedType; import com.facebook.presto.spi.type.NamedTypeSignature; import com.facebook.presto.spi.type.ParameterKind; import com.facebook.presto.spi.type.ParametricType; @@ -26,6 +27,7 @@ import com.facebook.presto.spi.type.TypeSignatureParameter; import java.util.List; +import java.util.Optional; import static com.google.common.base.Preconditions.checkArgument; import static java.util.stream.Collectors.toList; @@ -48,7 +50,9 @@ public String getName() @Override public Type createType(TypeManager typeManager, List parameters) { - checkArgument(!parameters.isEmpty(), "Row type must have at least one parameter"); + if (parameters.isEmpty()) { + parameters.add(TypeParameter.of(new NamedType(Optional.of(new RowFieldName(UnknownType.NAME, false)), UnknownType.UNKNOWN))); + } checkArgument( parameters.stream().allMatch(parameter -> parameter.getKind() == ParameterKind.NAMED_TYPE), "Expected only named types as a parameters, got %s", diff --git a/presto-main/src/main/resources/webapp/tableau/presto-client.js b/presto-main/src/main/resources/webapp/tableau/presto-client.js index 3bc357dd1b659..14de90193895b 100644 --- a/presto-main/src/main/resources/webapp/tableau/presto-client.js +++ b/presto-main/src/main/resources/webapp/tableau/presto-client.js @@ -17,6 +17,8 @@ function StatementClient(connectionData, headerCallback, dataCallback, errorCall this.currentResults = null; this.valid = true; + this.isHttps = window.location.protocol === "https:" + if (!(connectionData.sessionParameters === undefined)) { var parameterMap = JSON.parse(connectionData.sessionParameters); for (var name in parameterMap) { @@ -72,7 +74,7 @@ StatementClient.prototype.advance = function(lastRecordNumber) { var statementClient = this; $.ajax({ type: "GET", - url: this.currentResults.nextUri, + url: this.isHttps ? this.currentResults.nextUri.replace(/^http:/, 'https:') : this.currentResults.nextUri, headers: this.headers, dataType: 'json', // FIXME having problems when async: true diff --git a/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java b/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java index e552b3cfc5005..96204722bdce2 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java +++ b/presto-main/src/test/java/com/facebook/presto/server/remotetask/TestHttpRemoteTask.java @@ -38,6 +38,7 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.planner.Symbol; import com.facebook.presto.sql.planner.plan.PlanNodeId; import com.facebook.presto.testing.TestingHandleResolver; import com.facebook.presto.testing.TestingSplit; @@ -237,6 +238,8 @@ public void configure(Binder binder) jsonCodecBinder(binder).bindJsonCodec(TaskStatus.class); jsonCodecBinder(binder).bindJsonCodec(TaskInfo.class); jsonCodecBinder(binder).bindJsonCodec(TaskUpdateRequest.class); + jsonBinder(binder).addKeySerializerBinding(Symbol.class).to(Symbol.SymbolKeySerializer.class); + jsonBinder(binder).addKeyDeserializerBinding(Symbol.class).to(Symbol.SymbolKeyDeserializer.class); } @Provides diff --git a/presto-main/src/test/java/com/facebook/presto/server/security/TestSecurityConfig.java b/presto-main/src/test/java/com/facebook/presto/server/security/TestSecurityConfig.java index 96beefb828f7c..52064739490e7 100644 --- a/presto-main/src/test/java/com/facebook/presto/server/security/TestSecurityConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/server/security/TestSecurityConfig.java @@ -29,7 +29,8 @@ public class TestSecurityConfig public void testDefaults() { ConfigAssertions.assertRecordedDefaults(ConfigAssertions.recordDefaults(SecurityConfig.class) - .setAuthenticationTypes("")); + .setAuthenticationTypes("") + .setHttpAuthenticationPathRegex("^\b$")); } @Test @@ -37,10 +38,12 @@ public void testExplicitPropertyMappings() { Map properties = new ImmutableMap.Builder() .put("http-server.authentication.type", "KERBEROS,PASSWORD") + .put("http-server.http.authentication.path.regex", "^/v1/statement") .build(); SecurityConfig expected = new SecurityConfig() - .setAuthenticationTypes(ImmutableList.of(KERBEROS, PASSWORD)); + .setAuthenticationTypes(ImmutableList.of(KERBEROS, PASSWORD)) + .setHttpAuthenticationPathRegex("^/v1/statement"); ConfigAssertions.assertFullMapping(properties, expected); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestTypeValidator.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestTypeValidator.java index 29c42c9b586d7..1cf2a2cfd2591 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestTypeValidator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestTypeValidator.java @@ -219,7 +219,7 @@ public void testValidTypeOnlyCoercion() assertTypesValid(node); } - @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "type of symbol 'expr(_[0-9]+)?' is expected to be bigint, but the actual type is integer") + @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "type of symbol 'Symbol\\{name=expr(_[0-9]+)?, fields=\\[\\]\\}' is expected to be bigint, but the actual type is integer") public void testInvalidProject() { Expression expression1 = new Cast(columnB.toSymbolReference(), StandardTypes.INTEGER); @@ -236,7 +236,7 @@ public void testInvalidProject() assertTypesValid(node); } - @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "type of symbol 'sum(_[0-9]+)?' is expected to be double, but the actual type is bigint") + @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "type of symbol 'Symbol\\{name=sum(_[0-9]+)?, fields=\\[\\]\\}' is expected to be double, but the actual type is bigint") public void testInvalidAggregationFunctionCall() { Symbol aggregationSymbol = symbolAllocator.newSymbol("sum", DOUBLE); @@ -264,7 +264,7 @@ public void testInvalidAggregationFunctionCall() assertTypesValid(node); } - @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "type of symbol 'sum(_[0-9]+)?' is expected to be double, but the actual type is bigint") + @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "type of symbol 'Symbol\\{name=sum(_[0-9]+)?, fields=\\[\\]\\}' is expected to be double, but the actual type is bigint") public void testInvalidAggregationFunctionSignature() { Symbol aggregationSymbol = symbolAllocator.newSymbol("sum", DOUBLE); @@ -292,7 +292,7 @@ public void testInvalidAggregationFunctionSignature() assertTypesValid(node); } - @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "type of symbol 'sum(_[0-9]+)?' is expected to be double, but the actual type is bigint") + @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "type of symbol 'Symbol\\{name=sum(_[0-9]+)?, fields=\\[\\]\\}' is expected to be double, but the actual type is bigint") public void testInvalidWindowFunctionCall() { Symbol windowSymbol = symbolAllocator.newSymbol("sum", DOUBLE); @@ -331,7 +331,7 @@ public void testInvalidWindowFunctionCall() assertTypesValid(node); } - @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "type of symbol 'sum(_[0-9]+)?' is expected to be double, but the actual type is bigint") + @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "type of symbol 'Symbol\\{name=sum(_[0-9]+)?, fields=\\[\\]\\}' is expected to be double, but the actual type is bigint") public void testInvalidWindowFunctionSignature() { Symbol windowSymbol = symbolAllocator.newSymbol("sum", DOUBLE); @@ -370,7 +370,7 @@ public void testInvalidWindowFunctionSignature() assertTypesValid(node); } - @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "type of symbol 'output(_[0-9]+)?' is expected to be date, but the actual type is bigint") + @Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "type of symbol 'Symbol\\{name=output(_[0-9]+)?, fields=\\[\\]\\}' is expected to be date, but the actual type is bigint") public void testInvalidUnion() { Symbol outputSymbol = symbolAllocator.newSymbol("output", DATE); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/ExpressionMatcher.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/ExpressionMatcher.java index 83b363b1a303d..97101178f95a9 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/ExpressionMatcher.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/ExpressionMatcher.java @@ -20,16 +20,19 @@ import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.PlanNode; import com.facebook.presto.sql.planner.plan.ProjectNode; +import com.facebook.presto.sql.planner.plan.TableScanNode; import com.facebook.presto.sql.tree.Expression; import com.google.common.collect.ImmutableList; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Function; import java.util.stream.Collectors; import static com.facebook.presto.sql.ExpressionUtils.rewriteIdentifiersToSymbolReferences; import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static java.util.Objects.requireNonNull; public class ExpressionMatcher @@ -44,6 +47,12 @@ public ExpressionMatcher(String expression) this.expression = expression(requireNonNull(expression)); } + public ExpressionMatcher(Expression expression) + { + this.sql = requireNonNull(expression.toString()); + this.expression = requireNonNull(expression); + } + private Expression expression(String sql) { SqlParser parser = new SqlParser(); @@ -86,6 +95,10 @@ else if (node instanceof ApplyNode) { ApplyNode applyNode = (ApplyNode) node; return applyNode.getSubqueryAssignments().getMap(); } + else if (node instanceof TableScanNode) { + TableScanNode tableScanNode = (TableScanNode) node; + return tableScanNode.getAssignments().keySet().stream().collect(toImmutableMap(Function.identity(), Symbol::toSymbolReference)); + } else { return null; } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/ExpressionVerifier.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/ExpressionVerifier.java index 81aae0689f841..09d16bb038622 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/ExpressionVerifier.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/ExpressionVerifier.java @@ -21,10 +21,12 @@ import com.facebook.presto.sql.tree.CoalesceExpression; import com.facebook.presto.sql.tree.ComparisonExpression; import com.facebook.presto.sql.tree.DecimalLiteral; +import com.facebook.presto.sql.tree.DereferenceExpression; import com.facebook.presto.sql.tree.DoubleLiteral; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.FunctionCall; import com.facebook.presto.sql.tree.GenericLiteral; +import com.facebook.presto.sql.tree.Identifier; import com.facebook.presto.sql.tree.InListExpression; import com.facebook.presto.sql.tree.InPredicate; import com.facebook.presto.sql.tree.IsNotNullPredicate; @@ -307,13 +309,31 @@ protected Boolean visitNotExpression(NotExpression actual, Node expected) return false; } + @Override + protected Boolean visitIdentifier(Identifier actual, Node expected) + { + if (!(expected instanceof Identifier)) { + return false; + } + return actual.getValue().equals(((Identifier) expected).getValue()); + } + + @Override + protected Boolean visitDereferenceExpression(DereferenceExpression actual, Node expected) + { + if (!(expected instanceof DereferenceExpression)) { + return false; + } + return process(actual.getBase(), ((DereferenceExpression) expected).getBase()) && process(actual.getField(), ((DereferenceExpression) expected).getField()); + } + @Override protected Boolean visitSymbolReference(SymbolReference actual, Node expected) { if (!(expected instanceof SymbolReference)) { return false; } - return symbolAliases.get(((SymbolReference) expected).getName()).equals(actual); + return actual.equals(expected) || symbolAliases.get(((SymbolReference) expected).getName()).equals(actual); } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchPattern.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchPattern.java index 7f345a80362fb..bd160ce595048 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchPattern.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchPattern.java @@ -16,6 +16,7 @@ import com.facebook.presto.Session; import com.facebook.presto.cost.StatsProvider; import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.block.SortOrder; import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.sql.parser.ParsingOptions; @@ -134,6 +135,11 @@ public static PlanMatchPattern tableScan(String expectedTableName, String origin .build(); } + public static PlanMatchPattern tableScan(String expectedTableName, Collection columnHandles) + { + return TableScanMatcher.create(expectedTableName, columnHandles); + } + public static PlanMatchPattern tableScan(String expectedTableName, Map columnReferences) { PlanMatchPattern result = tableScan(expectedTableName); @@ -636,6 +642,11 @@ public static ExpressionMatcher expression(String expression) return new ExpressionMatcher(expression); } + public static ExpressionMatcher expression(Expression expression) + { + return new ExpressionMatcher(expression); + } + public PlanMatchPattern withOutputs(String... aliases) { return withOutputs(ImmutableList.copyOf(aliases)); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/SymbolAliases.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/SymbolAliases.java index 1ec113d359b06..4424c178767a6 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/SymbolAliases.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/SymbolAliases.java @@ -211,6 +211,12 @@ public Builder put(String alias, SymbolReference symbolReference) return this; } + // allow to rebind with a narrowed SymbolReference + if (bindings.containsKey(alias) && bindings.get(alias).contains(symbolReference)) { + bindings.put(alias, symbolReference); + return this; + } + checkState(!bindings.containsKey(alias), "Alias '%s' already bound to expression '%s'. Tried to rebind to '%s'", alias, bindings.get(alias), symbolReference); bindings.put(alias, symbolReference); return this; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/TableScanMatcher.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/TableScanMatcher.java index 9588d474f6060..83d0a0314a4ee 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/TableScanMatcher.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/TableScanMatcher.java @@ -17,15 +17,20 @@ import com.facebook.presto.cost.StatsProvider; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.TableMetadata; +import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.predicate.Domain; import com.facebook.presto.sql.planner.Symbol; import com.facebook.presto.sql.planner.plan.PlanNode; import com.facebook.presto.sql.planner.plan.TableScanNode; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.SymbolReference; +import com.google.common.collect.ImmutableSet; +import java.util.Collection; import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.function.Function; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; import static com.facebook.presto.sql.planner.assertions.Util.domainsMatch; @@ -41,13 +46,15 @@ final class TableScanMatcher private final Optional> expectedConstraint; private final Optional expectedOriginalConstraint; private final Optional hasTableLayout; + private final Optional> expectedColumnHandles; - private TableScanMatcher(String expectedTableName, Optional> expectedConstraint, Optional originalConstraint, Optional hasTableLayout) + private TableScanMatcher(String expectedTableName, Optional> expectedConstraint, Optional originalConstraint, Optional hasTableLayout, Optional> expectedColumnHandles) { this.expectedTableName = requireNonNull(expectedTableName, "expectedTableName is null"); this.expectedConstraint = requireNonNull(expectedConstraint, "expectedConstraint is null"); this.expectedOriginalConstraint = requireNonNull(originalConstraint, "expectedOriginalConstraint is null"); this.hasTableLayout = requireNonNull(hasTableLayout, "hasTableLayout is null"); + this.expectedColumnHandles = requireNonNull(expectedColumnHandles, "expectedColumnHandles is null"); } @Override @@ -69,7 +76,9 @@ public MatchResult detailMatches(PlanNode node, StatsProvider stats, Session ses originalConstraintMatches(tableScanNode) && ((!expectedConstraint.isPresent()) || domainsMatch(expectedConstraint, tableScanNode.getCurrentConstraint(), tableScanNode.getTable(), session, metadata)) && - hasTableLayout(tableScanNode)); + hasTableLayout(tableScanNode) && + ((!expectedColumnHandles.isPresent()) || + expectedColumnHandles.get().equals(ImmutableSet.copyOf(tableScanNode.getAssignments().values())))); } private boolean originalConstraintMatches(TableScanNode node) @@ -90,6 +99,11 @@ private boolean hasTableLayout(TableScanNode tableScanNode) return !hasTableLayout.isPresent() || hasTableLayout.get() == tableScanNode.getLayout().isPresent(); } + public static Function> actualColumnHandles() + { + return node -> ((TableScanNode) node).getAssignments().values(); + } + @Override public String toString() { @@ -99,6 +113,7 @@ public String toString() .add("expectedConstraint", expectedConstraint.orElse(null)) .add("expectedOriginalConstraint", expectedOriginalConstraint.orElse(null)) .add("hasTableLayout", hasTableLayout.orElse(null)) + .add("expectedColumnHandles", expectedColumnHandles.orElse(null)) .toString(); } @@ -112,12 +127,20 @@ public static PlanMatchPattern create(String expectedTableName) return builder(expectedTableName).build(); } + public static PlanMatchPattern create(String expectedTableName, Collection columnHandles) + { + return builder(expectedTableName) + .expectedColumnHandles(columnHandles) + .build(); + } + public static class Builder { private final String expectedTableName; private Optional> expectedConstraint = Optional.empty(); private Optional expectedOriginalConstraint = Optional.empty(); private Optional hasTableLayout = Optional.empty(); + private Optional> expectedColumnHandles = Optional.empty(); private Builder(String expectedTableName) { @@ -142,6 +165,12 @@ public Builder hasTableLayout() return this; } + public Builder expectedColumnHandles(Collection expectedColumnHandles) + { + this.expectedColumnHandles = Optional.of(ImmutableSet.copyOf(expectedColumnHandles)); + return this; + } + PlanMatchPattern build() { PlanMatchPattern result = node(TableScanNode.class).with( @@ -149,7 +178,8 @@ PlanMatchPattern build() expectedTableName, expectedConstraint, expectedOriginalConstraint, - hasTableLayout)); + hasTableLayout, + expectedColumnHandles)); return result; } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPickColumnHandles.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPickColumnHandles.java new file mode 100644 index 0000000000000..b16dedb3944d5 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPickColumnHandles.java @@ -0,0 +1,139 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.Session; +import com.facebook.presto.connector.ConnectorId; +import com.facebook.presto.metadata.AbstractMockMetadata; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.metadata.TableHandle; +import com.facebook.presto.metadata.TableLayout; +import com.facebook.presto.metadata.TableLayoutHandle; +import com.facebook.presto.metadata.TableLayoutResult; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorTableLayout; +import com.facebook.presto.spi.Constraint; +import com.facebook.presto.spi.predicate.TupleDomain; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.sql.planner.Symbol; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import com.facebook.presto.sql.tree.SymbolReference; +import com.facebook.presto.tpch.TpchColumnHandle; +import com.facebook.presto.tpch.TpchTableHandle; +import com.facebook.presto.tpch.TpchTableLayoutHandle; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static com.facebook.presto.spi.predicate.TupleDomain.all; +import static com.facebook.presto.spi.predicate.TupleDomain.none; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.RowType.field; +import static com.facebook.presto.spi.type.RowType.from; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.expression; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.tableScan; +import static com.facebook.presto.tpch.TpchMetadata.TINY_SCALE_FACTOR; +import static com.facebook.presto.tpch.TpchTransactionHandle.INSTANCE; +import static java.util.Collections.emptyList; +import static java.util.Objects.requireNonNull; + +public class TestPickColumnHandles + extends BaseRuleTest +{ + private static TpchTableHandle tableHandle = new TpchTableHandle("local", "orders", TINY_SCALE_FACTOR); + private static TpchTableLayoutHandle tableLayoutHandle = new TpchTableLayoutHandle(tableHandle, all()); + private static ConnectorId local = new ConnectorId("local"); + private static TableLayoutHandle tableLayout = new TableLayoutHandle(local, INSTANCE, tableLayoutHandle); + private static Type columnType = from(ImmutableList.of(field("x", BIGINT), field("y", BIGINT))); + private static TableHandle table = new TableHandle(local, tableHandle); + + @Test + public void testColumnReplacedWithNewHandle() + { + List columns = ImmutableList.of(new TpchColumnHandle("totalprice_new", BIGINT)); + Metadata mockMetadata = new MockMetadata(ImmutableList.of( + new TableLayoutResult(new TableLayout(tableLayout, getConnectorTableLayout(columns)), none()))); + tester().assertThat(new PickColumnHandles(mockMetadata)) + .on(p -> { + Symbol fakeTotalpriceWithFields = p.symbol("totalprice", columnType, ImmutableSet.of("y")); + return p.tableScan( + table, + ImmutableList.of(fakeTotalpriceWithFields), + ImmutableMap.of(fakeTotalpriceWithFields, new TpchColumnHandle("totalprice", columnType))); + }) + .matches(tableScan("orders", columns) + .withExactAssignedOutputs(expression(new SymbolReference("totalprice", ImmutableSet.of("y"))))); + } + + @Test + public void testMetadataNotGiveNewHandle() + { + tester().assertThat(new PickColumnHandles(tester().getMetadata())) + .on(p -> { + Symbol fakeTotalpriceWithFields = p.symbol("totalprice", columnType, ImmutableSet.of("y")); + return p.tableScan( + table, + ImmutableList.of(fakeTotalpriceWithFields), + ImmutableMap.of(fakeTotalpriceWithFields, new TpchColumnHandle("totalprice", columnType))); + }) + .doesNotFire(); + } + + @Test + public void testAllNestedFieldReferenced() + { + tester().assertThat(new PickColumnHandles(tester().getMetadata())) + .on(p -> { + Symbol fakeTotalprice = p.symbol("totalprice", columnType); + return p.tableScan( + table, + ImmutableList.of(fakeTotalprice), + ImmutableMap.of(fakeTotalprice, new TpchColumnHandle("totalprice", columnType))); + }) + .doesNotFire(); + } + + private static class MockMetadata + extends AbstractMockMetadata + { + private final List expectedLayouts; + + public MockMetadata(List expectedLayouts) + { + this.expectedLayouts = requireNonNull(expectedLayouts, "expectedLayouts is null"); + } + + @Override + public List getLayouts(Session session, TableHandle tableHandle, Constraint constraint, Optional> desiredColumns) + { + return expectedLayouts; + } + } + + private static ConnectorTableLayout getConnectorTableLayout(List columns) + { + return new ConnectorTableLayout(tableLayoutHandle, + Optional.of(columns), + TupleDomain.all(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + emptyList()); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneFilterColumnFields.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneFilterColumnFields.java new file mode 100644 index 0000000000000..64775f2baa606 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneFilterColumnFields.java @@ -0,0 +1,89 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.sql.planner.Symbol; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; +import com.facebook.presto.sql.planner.plan.Assignments; +import com.facebook.presto.sql.planner.plan.ProjectNode; +import com.facebook.presto.sql.tree.DereferenceExpression; +import com.facebook.presto.sql.tree.Identifier; +import com.facebook.presto.sql.tree.SymbolReference; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.testng.annotations.Test; + +import java.util.function.Predicate; + +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.RowType.field; +import static com.facebook.presto.spi.type.RowType.from; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.expression; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.filter; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.strictProject; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; + +public class TestPruneFilterColumnFields + extends BaseRuleTest +{ + @Test + public void testNotAllNestedFieldsReferenced() + { + tester().assertThat(new PruneFilterColumnFields()) + .on(p -> buildProjectedFilter(p, symbol -> symbol.getName().equals("expr_y"))) + .matches( + strictProject( + ImmutableMap.of("expr", expression("a.y")), + filter( + "a.x > 5", + strictProject( + ImmutableMap.of("a", expression(new SymbolReference("a", ImmutableSet.of("x", "y")))), + values("a"))))); + } + + @Test + public void testWholeRowReferenced() + { + tester().assertThat(new PruneFilterColumns()) + .on(p -> buildProjectedFilter(p, symbol -> symbol.getName().equals("a"))) + .doesNotFire(); + } + + @Test + public void testWholeRowAndNestedFieldsReferenced() + { + tester().assertThat(new PruneFilterColumns()) + .on(p -> buildProjectedFilter(p, symbol -> true)) + .doesNotFire(); + } + + private ProjectNode buildProjectedFilter(PlanBuilder planBuilder, Predicate projectionFilter) + { + Symbol a = planBuilder.symbol("a", from(ImmutableList.of(field("x", BIGINT), field("y", BIGINT), field("z", BIGINT)))); + Assignments assignments = Assignments.builder() + .put(a, a.toSymbolReference()) + .put(planBuilder.symbol("expr_x"), new DereferenceExpression(a.toSymbolReference(), new Identifier("x"))) + .put(planBuilder.symbol("expr_y"), new DereferenceExpression(a.toSymbolReference(), new Identifier("y"))) + .put(planBuilder.symbol("expr_z"), new DereferenceExpression(a.toSymbolReference(), new Identifier("z"))) + .build(); + + return planBuilder.project( + assignments.filter(projectionFilter), + planBuilder.filter( + planBuilder.expression("a.x > 5"), + planBuilder.values(a))); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneProjectColumnFields.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneProjectColumnFields.java new file mode 100644 index 0000000000000..020bb7c53b643 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneProjectColumnFields.java @@ -0,0 +1,96 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.sql.planner.Symbol; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; +import com.facebook.presto.sql.planner.plan.Assignments; +import com.facebook.presto.sql.planner.plan.ProjectNode; +import com.facebook.presto.sql.tree.DereferenceExpression; +import com.facebook.presto.sql.tree.Identifier; +import com.facebook.presto.sql.tree.SymbolReference; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.testng.annotations.Test; + +import java.util.function.Predicate; + +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.RowType.field; +import static com.facebook.presto.spi.type.RowType.from; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.expression; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.strictProject; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; + +public class TestPruneProjectColumnFields + extends BaseRuleTest +{ + @Test + public void testNotAllNestedFieldsReferenced() + { + tester().assertThat(new PruneProjectColumnFields()) + .on(p -> buildProject(p, symbol -> symbol.getName().equals("expr_y"))) + .matches( + strictProject( + ImmutableMap.of("expr", expression("a.y")), + strictProject( + ImmutableMap.of("a", expression(new SymbolReference("a", ImmutableSet.of("y")))), + values("a")))); + } + + @Test + public void testMixIntermediateRowAndLeafFieldReferenced() + { + tester().assertThat(new PruneProjectColumnFields()) + .on(p -> buildProject(p, symbol -> symbol.getName().equals("expr_y") || symbol.getName().equals("expr_i"))) + .matches( + strictProject( + ImmutableMap.of("expr_y", expression("a.y"), "expr_i", expression("a.y.i")), + strictProject( + ImmutableMap.of("a", expression(new SymbolReference("a", ImmutableSet.of("y")))), + values("a")))); + } + + @Test + public void testWholeRowReferenced() + { + tester().assertThat(new PruneProjectColumnFields()) + .on(p -> buildProject(p, symbol -> symbol.getName().equals("a"))) + .doesNotFire(); + } + + private ProjectNode buildProject(PlanBuilder planBuilder, Predicate projectionFilter) + { + Type nestedFieldType = from(ImmutableList.of(field("i", BIGINT), field("j", BIGINT))); + Type columnType = from(ImmutableList.of(field("x", BIGINT), field("y", nestedFieldType))); + Symbol a = planBuilder.symbol("a", columnType); + Symbol y = planBuilder.symbol("expr_y", nestedFieldType); + Symbol i = planBuilder.symbol("expr_i"); + DereferenceExpression expY = new DereferenceExpression(a.toSymbolReference(), new Identifier("y")); + DereferenceExpression expI = new DereferenceExpression(expY, new Identifier("i")); + Assignments assignments = Assignments.builder() + .put(a, a.toSymbolReference()) + .put(y, expY) + .put(i, expI) + .build(); + return planBuilder.project( + assignments.filter(projectionFilter), + planBuilder.project( + Assignments.identity(a), + planBuilder.values(a))); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneTableScanColumnFields.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneTableScanColumnFields.java new file mode 100644 index 0000000000000..257832193b44f --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestPruneTableScanColumnFields.java @@ -0,0 +1,102 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.connector.ConnectorId; +import com.facebook.presto.metadata.TableHandle; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.sql.planner.Symbol; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder; +import com.facebook.presto.sql.planner.plan.Assignments; +import com.facebook.presto.sql.planner.plan.ProjectNode; +import com.facebook.presto.sql.tree.DereferenceExpression; +import com.facebook.presto.sql.tree.Identifier; +import com.facebook.presto.sql.tree.SymbolReference; +import com.facebook.presto.tpch.TpchColumnHandle; +import com.facebook.presto.tpch.TpchTableHandle; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.testng.annotations.Test; + +import java.util.function.Predicate; + +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.RowType.field; +import static com.facebook.presto.spi.type.RowType.from; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.expression; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.strictProject; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.strictTableScan; +import static com.facebook.presto.tpch.TpchMetadata.TINY_SCALE_FACTOR; + +public class TestPruneTableScanColumnFields + extends BaseRuleTest +{ + @Test + public void testNotAllOutputsReferenced() + { + tester().assertThat(new PruneTableScanColumnFields()) + .on(p -> buildProjectedTableScan(p, symbol -> symbol.getName().equals("expr_y"))) + .matches( + strictProject( + ImmutableMap.of("expr_y", expression("totalprice.y")), + strictTableScan("orders", ImmutableMap.of("totalprice_", "totalprice")) + .withExactAssignedOutputs(expression(new SymbolReference("totalprice", ImmutableSet.of("y")))))); + } + + @Test + public void testMixIntermediateRowAndLeafFieldReferenced() + { + tester().assertThat(new PruneTableScanColumnFields()) + .on(p -> buildProjectedTableScan(p, symbol -> symbol.getName().equals("expr_y") || symbol.getName().equals("expr_i"))) + .matches( + strictProject( + ImmutableMap.of("expr_y", expression("totalprice.y"), "expr_i", expression("totalprice.y.i")), + strictTableScan("orders", ImmutableMap.of("totalprice_", "totalprice")) + .withExactAssignedOutputs(expression(new SymbolReference("totalprice", ImmutableSet.of("y")))))); + } + + @Test + public void testAllOutputsReferenced() + { + tester().assertThat(new PruneTableScanColumnFields()) + .on(p -> buildProjectedTableScan(p, symbol -> symbol.getName().equals("totalprice"))) + .doesNotFire(); + } + + private ProjectNode buildProjectedTableScan(PlanBuilder planBuilder, Predicate projectionFilter) + { + Type nestedFieldType = from(ImmutableList.of(field("i", BIGINT), field("j", BIGINT))); + Type columnType = from(ImmutableList.of(field("x", BIGINT), field("y", nestedFieldType))); + Symbol fakeTotalprice = planBuilder.symbol("totalprice", columnType); + Symbol y = planBuilder.symbol("expr_y"); + Symbol i = planBuilder.symbol("expr_i"); + DereferenceExpression expY = new DereferenceExpression(fakeTotalprice.toSymbolReference(), new Identifier("y")); + DereferenceExpression expI = new DereferenceExpression(expY, new Identifier("i")); + Assignments assignments = Assignments.builder() + .put(fakeTotalprice, fakeTotalprice.toSymbolReference()) + .put(y, expY) + .put(i, expI) + .build(); + return planBuilder.project( + assignments.filter(projectionFilter), + planBuilder.tableScan( + new TableHandle( + new ConnectorId("local"), + new TpchTableHandle("local", "orders", TINY_SCALE_FACTOR)), + ImmutableList.of(fakeTotalprice), + ImmutableMap.of(fakeTotalprice, new TpchColumnHandle("totalprice", columnType)))); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java index fa159aa44312b..ae966ef0e58cc 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java @@ -685,7 +685,12 @@ public Symbol symbol(String name) public Symbol symbol(String name, Type type) { - Symbol symbol = new Symbol(name); + return symbol(name, type, ImmutableSet.of()); + } + + public Symbol symbol(String name, Type type, Set fields) + { + Symbol symbol = new Symbol(name, fields); Type old = symbols.put(symbol, type); if (old != null && !old.equals(type)) { diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestWindowNode.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestWindowNode.java index 12c8a9537a2e2..a7e2fa87240d7 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestWindowNode.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestWindowNode.java @@ -67,6 +67,8 @@ Slice.class, new SliceSerializer(), Slice.class, new SliceDeserializer(), Expression.class, new Serialization.ExpressionDeserializer(sqlParser), FunctionCall.class, new Serialization.FunctionCallDeserializer(sqlParser))); + provider.setKeySerializers(ImmutableMap.of(Symbol.class, new Symbol.SymbolKeySerializer())); + provider.setKeyDeserializers(ImmutableMap.of(Symbol.class, new Symbol.SymbolKeyDeserializer())); objectMapper = provider.get(); } diff --git a/presto-matching/pom.xml b/presto-matching/pom.xml index e8df6cfb48bc4..f51fc99922e2b 100644 --- a/presto-matching/pom.xml +++ b/presto-matching/pom.xml @@ -18,7 +18,7 @@ presto-root com.facebook.presto - 0.208 + 0.208-tw-0.53 presto-matching diff --git a/presto-memory-context/pom.xml b/presto-memory-context/pom.xml index 3c52d4e70404a..39c1506c76cb5 100644 --- a/presto-memory-context/pom.xml +++ b/presto-memory-context/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-memory-context diff --git a/presto-memory/pom.xml b/presto-memory/pom.xml index 1bd12ce390b08..3437b2bfe3ad7 100644 --- a/presto-memory/pom.xml +++ b/presto-memory/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-memory diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index ec72b39e5eada..14c9eacfd8a9a 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-ml diff --git a/presto-mongodb/pom.xml b/presto-mongodb/pom.xml index 08c3f0c9fbf37..44e992baf3a57 100644 --- a/presto-mongodb/pom.xml +++ b/presto-mongodb/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-mongodb diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 36f00f239d855..fe687e6ec2263 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-mysql diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 67e06d896d015..9272b793dca6f 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-orc diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index b799cde2883f7..cc74458d40ec2 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-parser diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/SymbolReference.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/SymbolReference.java index 51b88d4dcf3a6..f1832c56f0f74 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/tree/SymbolReference.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/SymbolReference.java @@ -14,20 +14,31 @@ package com.facebook.presto.sql.tree; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; public class SymbolReference extends Expression { private final String name; + private final Set fields; public SymbolReference(String name) { super(Optional.empty()); this.name = name; + this.fields = ImmutableSet.of(); + } + + public SymbolReference(String name, Set fields) + { + super(Optional.empty()); + this.name = name; + this.fields = ImmutableSet.copyOf(fields); } public String getName() @@ -35,6 +46,11 @@ public String getName() return name; } + public Set getFields() + { + return fields; + } + @Override public R accept(AstVisitor visitor, C context) { @@ -57,12 +73,29 @@ public boolean equals(Object o) return false; } SymbolReference that = (SymbolReference) o; - return Objects.equals(name, that.name); + return Objects.equals(name, that.name) && Objects.equals(fields, that.fields); } @Override public int hashCode() { - return Objects.hash(name); + return Objects.hash(name, fields); + } + + public boolean contains(SymbolReference o) + { + if (!this.getName().equals(o.getName())) { + return false; + } + + if (this.getFields().isEmpty()) { + return true; + } + + if (o.getFields().isEmpty()) { + return false; + } + + return this.getFields().containsAll(o.getFields()); } } diff --git a/presto-password-authenticators/pom.xml b/presto-password-authenticators/pom.xml index 9b9529d802429..24299c6ecd740 100644 --- a/presto-password-authenticators/pom.xml +++ b/presto-password-authenticators/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-password-authenticators diff --git a/presto-plugin-toolkit/pom.xml b/presto-plugin-toolkit/pom.xml index faefc9fec5ce2..3e89e93dab4a4 100644 --- a/presto-plugin-toolkit/pom.xml +++ b/presto-plugin-toolkit/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-plugin-toolkit diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index d3fa627467a9c..13c45340e3ccf 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-postgresql diff --git a/presto-product-tests/conf/presto/etc/singlenode-kerberized.properties b/presto-product-tests/conf/presto/etc/singlenode-kerberized.properties index da8640e97ed4f..4d842152dd7c1 100644 --- a/presto-product-tests/conf/presto/etc/singlenode-kerberized.properties +++ b/presto-product-tests/conf/presto/etc/singlenode-kerberized.properties @@ -19,7 +19,7 @@ discovery.uri=https://presto-master.docker.cluster:7778 http.authentication.krb5.config=/etc/krb5.conf http-server.authentication.type=KERBEROS,CERTIFICATE -http.server.authentication.krb5.service-name=presto-server +http.server.authentication.krb5.service-name=presto-server/presto-master.docker.cluster@LABS.TERADATA.COM http-server.http.enabled=false http-server.https.enabled=true http-server.https.port=7778 diff --git a/presto-product-tests/conf/tempto/tempto-configuration-for-docker-kerberos.yaml b/presto-product-tests/conf/tempto/tempto-configuration-for-docker-kerberos.yaml index 6e9c06e2a0403..97366c081826b 100644 --- a/presto-product-tests/conf/tempto/tempto-configuration-for-docker-kerberos.yaml +++ b/presto-product-tests/conf/tempto/tempto-configuration-for-docker-kerberos.yaml @@ -39,7 +39,7 @@ databases: cli_kerberos_principal: presto-client/presto-master.docker.cluster@LABS.TERADATA.COM cli_kerberos_keytab: /etc/presto/conf/presto-client.keytab cli_kerberos_config_path: /etc/krb5.conf - cli_kerberos_service_name: presto-server + cli_kerberos_service_name: presto-server/presto-master.docker.cluster@LABS.TERADATA.COM cli_kerberos_use_canonical_hostname: false configured_hdfs_user: hdfs diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 8959c496eb5b1..e2e394b43ed0e 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.208 + 0.208-tw-0.53 presto-product-tests diff --git a/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestHiveCoercion.java b/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestHiveCoercion.java index 7adc0335ef51e..b82b1cb5ddcfd 100644 --- a/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestHiveCoercion.java +++ b/presto-product-tests/src/main/java/com/facebook/presto/tests/hive/TestHiveCoercion.java @@ -256,7 +256,6 @@ public void testHiveCoercionAvro() private void doTestHiveCoercion(HiveTableDefinition tableDefinition) { String tableName = mutableTableInstanceOf(tableDefinition).getNameInDatabase(); - String floatToDoubleType = tableName.toLowerCase(Locale.ENGLISH).contains("parquet") ? "DOUBLE" : "REAL"; query(format( diff --git a/presto-proxy/pom.xml b/presto-proxy/pom.xml index 2ab17d70d97ad..51748a035e836 100644 --- a/presto-proxy/pom.xml +++ b/presto-proxy/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-proxy diff --git a/presto-raptor/pom.xml b/presto-raptor/pom.xml index e8a4ec7696e45..eabad7a1e3160 100644 --- a/presto-raptor/pom.xml +++ b/presto-raptor/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-raptor diff --git a/presto-rcfile/pom.xml b/presto-rcfile/pom.xml index ca9a530887f4f..343b060d5f4ac 100644 --- a/presto-rcfile/pom.xml +++ b/presto-rcfile/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-rcfile diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 4a4fe835a1c3d..9ef329b09134a 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-record-decoder @@ -67,6 +67,18 @@ jackson-annotations + + + org.apache.thrift + libthrift + + + + commons-lang + commons-lang + 2.5 + + org.testng @@ -79,11 +91,58 @@ testing test - + + com.facebook.presto + presto-main + test + + + javax.servlet-api + javax.servlet + + + org.assertj assertj-core test + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + org.apache.thrift.tools + maven-thrift-plugin + 0.1.11 + + /usr/local/bin/thrift + + + + thrift-sources + generate-sources + + compile + + + + thrift-test-sources + generate-test-sources + + testCompile + + + + + + diff --git a/presto-record-decoder/src/main/java/com/facebook/presto/decoder/DecoderModule.java b/presto-record-decoder/src/main/java/com/facebook/presto/decoder/DecoderModule.java index 1898308a39876..7d70be5fa7455 100644 --- a/presto-record-decoder/src/main/java/com/facebook/presto/decoder/DecoderModule.java +++ b/presto-record-decoder/src/main/java/com/facebook/presto/decoder/DecoderModule.java @@ -21,6 +21,8 @@ import com.facebook.presto.decoder.json.JsonRowDecoderFactory; import com.facebook.presto.decoder.raw.RawRowDecoder; import com.facebook.presto.decoder.raw.RawRowDecoderFactory; +import com.facebook.presto.decoder.thrift.ThriftRowDecoder; +import com.facebook.presto.decoder.thrift.ThriftRowDecoderFactory; import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.multibindings.MapBinder; @@ -41,6 +43,7 @@ public void configure(Binder binder) decoderFactoriesByName.addBinding(CsvRowDecoder.NAME).to(CsvRowDecoderFactory.class).in(SINGLETON); decoderFactoriesByName.addBinding(JsonRowDecoder.NAME).to(JsonRowDecoderFactory.class).in(SINGLETON); decoderFactoriesByName.addBinding(RawRowDecoder.NAME).to(RawRowDecoderFactory.class).in(SINGLETON); + decoderFactoriesByName.addBinding(ThriftRowDecoder.NAME).to(ThriftRowDecoderFactory.class).in(SINGLETON); binder.bind(DispatchingRowDecoderFactory.class).in(SINGLETON); } diff --git a/presto-record-decoder/src/main/java/com/facebook/presto/decoder/FieldValueProvider.java b/presto-record-decoder/src/main/java/com/facebook/presto/decoder/FieldValueProvider.java index 51b2d013710c2..bef50fd51833b 100644 --- a/presto-record-decoder/src/main/java/com/facebook/presto/decoder/FieldValueProvider.java +++ b/presto-record-decoder/src/main/java/com/facebook/presto/decoder/FieldValueProvider.java @@ -14,6 +14,7 @@ package com.facebook.presto.decoder; import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.block.Block; import io.airlift.slice.Slice; /** @@ -42,4 +43,9 @@ public Slice getSlice() } public abstract boolean isNull(); + + public Block getBlock() + { + throw new PrestoException(DecoderErrorCode.DECODER_CONVERSION_NOT_SUPPORTED, "conversion to block not supported"); + } } diff --git a/presto-record-decoder/src/main/java/com/facebook/presto/decoder/raw/RawColumnDecoder.java b/presto-record-decoder/src/main/java/com/facebook/presto/decoder/raw/RawColumnDecoder.java index a8c436d11895b..2e3153f255890 100644 --- a/presto-record-decoder/src/main/java/com/facebook/presto/decoder/raw/RawColumnDecoder.java +++ b/presto-record-decoder/src/main/java/com/facebook/presto/decoder/raw/RawColumnDecoder.java @@ -181,7 +181,7 @@ private void checkFieldTypeOneOf(FieldType declaredFieldType, String columnName, } } - public FieldValueProvider decodeField(byte[] value) + public FieldValueProvider decode(byte[] value) { requireNonNull(value, "value is null"); diff --git a/presto-record-decoder/src/main/java/com/facebook/presto/decoder/raw/RawRowDecoder.java b/presto-record-decoder/src/main/java/com/facebook/presto/decoder/raw/RawRowDecoder.java index f497b30867fa2..4028b1ca54d5d 100644 --- a/presto-record-decoder/src/main/java/com/facebook/presto/decoder/raw/RawRowDecoder.java +++ b/presto-record-decoder/src/main/java/com/facebook/presto/decoder/raw/RawRowDecoder.java @@ -53,6 +53,6 @@ public Optional> decodeRow(byte[] d return Optional.of(columnDecoders.entrySet().stream() .collect(toImmutableMap( Map.Entry::getKey, - entry -> entry.getValue().decodeField(data)))); + entry -> entry.getValue().decode(data)))); } } diff --git a/presto-record-decoder/src/main/java/com/facebook/presto/decoder/thrift/ThriftColumnDecoder.java b/presto-record-decoder/src/main/java/com/facebook/presto/decoder/thrift/ThriftColumnDecoder.java new file mode 100644 index 0000000000000..d7fbe31e1eefd --- /dev/null +++ b/presto-record-decoder/src/main/java/com/facebook/presto/decoder/thrift/ThriftColumnDecoder.java @@ -0,0 +1,381 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.decoder.thrift; + +import com.facebook.presto.decoder.DecoderColumnHandle; +import com.facebook.presto.decoder.FieldValueProvider; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.type.StandardTypes; +import com.facebook.presto.spi.type.Type; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; +import org.joda.time.DateTimeZone; + +import java.sql.Date; +import java.sql.Timestamp; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static com.facebook.presto.decoder.DecoderErrorCode.DECODER_CONVERSION_NOT_SUPPORTED; +import static com.facebook.presto.spi.StandardErrorCode.GENERIC_USER_ERROR; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.Chars.isCharType; +import static com.facebook.presto.spi.type.Chars.truncateToLengthAndTrimSpaces; +import static com.facebook.presto.spi.type.DateType.DATE; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.IntegerType.INTEGER; +import static com.facebook.presto.spi.type.RealType.REAL; +import static com.facebook.presto.spi.type.SmallintType.SMALLINT; +import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; +import static com.facebook.presto.spi.type.TinyintType.TINYINT; +import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; +import static com.facebook.presto.spi.type.Varchars.isVarcharType; +import static com.facebook.presto.spi.type.Varchars.truncateToLength; +import static com.google.common.base.Preconditions.checkArgument; +import static io.airlift.slice.Slices.EMPTY_SLICE; +import static java.lang.Float.floatToRawIntBits; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public class ThriftColumnDecoder +{ + private final String columnName; + private final Type columnType; + + public ThriftColumnDecoder(DecoderColumnHandle columnHandle) + { + try { + requireNonNull(columnHandle, "columnHandle is null"); + checkArgument(!columnHandle.isInternal(), "unexpected internal column '%s'", columnHandle.getName()); + columnName = columnHandle.getName(); + columnType = columnHandle.getType(); + } + catch (IllegalArgumentException e) { + throw new PrestoException(GENERIC_USER_ERROR, e); + } + } + + public FieldValueProvider decode(Object value) + { + return new ObjectValueProvider(value, columnName, columnType); + } + + @Override + public String toString() + { + return format("FieldDecoder[%s/%s]", ThriftRowDecoder.NAME, columnName); + } + + public static class ObjectValueProvider + extends FieldValueProvider + { + protected final Object value; + protected final String name; + protected final Type type; + + public ObjectValueProvider(Object value, String name, Type type) + { + this.value = value; + this.name = requireNonNull(name, "name is null"); + this.type = requireNonNull(type, "type is null"); + } + + @Override + public final boolean isNull() + { + return value == null; + } + + @Override + public boolean getBoolean() + { + try { + return isNull() ? false : (Boolean) value; + } + catch (RuntimeException e) { + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("could not parse value for column %s as %s", name, type), e); + } + } + + @Override + public long getLong() + { + try { + return isNull() ? 0L : getLongExpressedValue(value); + } + catch (RuntimeException e) { + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("could not parse value for column %s as %s", name, type), e); + } + } + + private static long getLongExpressedValue(Object value) + { + if (value instanceof Date) { + long storageTime = ((Date) value).getTime(); + // convert date from VM current time zone to UTC + long utcMillis = storageTime + DateTimeZone.getDefault().getOffset(storageTime); + return TimeUnit.MILLISECONDS.toDays(utcMillis); + } + if (value instanceof Timestamp) { + long parsedJvmMillis = ((Timestamp) value).getTime(); + DateTimeZone jvmTimeZone = DateTimeZone.getDefault(); + long convertedMillis = jvmTimeZone.convertUTCToLocal(parsedJvmMillis); + + return convertedMillis; + } + if (value instanceof Float) { + return floatToRawIntBits(((Float) value)); + } + return ((Number) value).longValue(); + } + + @Override + public double getDouble() + { + try { + return isNull() ? 0.0d : (Double) value; + } + catch (RuntimeException e) { + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("could not parse value for column %s as %s", name, type), e); + } + } + + @Override + public Slice getSlice() + { + try { + return isNull() ? EMPTY_SLICE : getSliceExpressedValue(value, type); + } + catch (RuntimeException e) { + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("could not parse value for column %s as %s", name, type), e); + } + } + + private static Slice getSliceExpressedValue(Object value, Type type) + { + Slice sliceValue; + if (value instanceof String) { + sliceValue = Slices.utf8Slice((String) value); + } + else if (value instanceof byte[]) { + sliceValue = Slices.wrappedBuffer((byte[]) value); + } + else if (value instanceof Integer) { + sliceValue = Slices.utf8Slice(value.toString()); + } + else { + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, "unsupported string field type: " + value.getClass().getName()); + } + if (isVarcharType(type)) { + sliceValue = truncateToLength(sliceValue, type); + } + if (isCharType(type)) { + sliceValue = truncateToLengthAndTrimSpaces(sliceValue, type); + } + + return sliceValue; + } + + @Override + public Block getBlock() + { + try { + if (isNull()) { + return null; + } + return serializeObject(type, null, value); + } + catch (RuntimeException e) { + throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("could not parse value for column %s as %s", name, type), e); + } + } + + private static Block serializeObject(Type type, BlockBuilder builder, Object object) + { + if (!isStructuralType(type)) { + serializePrimitive(type, builder, object); + return null; + } + else if (isArrayType(type)) { + return serializeList(type, builder, object); + } + else if (isMapType(type)) { + return serializeMap(type, builder, object); + } + else if (isRowType(type)) { + return serializeStruct(type, builder, object); + } + throw new RuntimeException("Unknown object type: " + type); + } + + private static Block serializeList(Type type, BlockBuilder builder, Object object) + { + List list = (List) object; + if (list == null) { + requireNonNull(builder, "parent builder is null").appendNull(); + return null; + } + + List typeParameters = type.getTypeParameters(); + checkArgument(typeParameters.size() == 1, "list must have exactly 1 type parameter"); + Type elementType = typeParameters.get(0); + + BlockBuilder currentBuilder; + if (builder != null) { + currentBuilder = builder.beginBlockEntry(); + } + else { + currentBuilder = elementType.createBlockBuilder(null, list.size()); + } + + for (Object element : list) { + serializeObject(elementType, currentBuilder, element); + } + + if (builder != null) { + builder.closeEntry(); + return null; + } + else { + Block resultBlock = currentBuilder.build(); + return resultBlock; + } + } + + private static Block serializeMap(Type type, BlockBuilder builder, Object object) + { + Map map = (Map) object; + if (map == null) { + requireNonNull(builder, "parent builder is null").appendNull(); + return null; + } + + List typeParameters = type.getTypeParameters(); + checkArgument(typeParameters.size() == 2, "map must have exactly 2 type parameter"); + Type keyType = typeParameters.get(0); + Type valueType = typeParameters.get(1); + + BlockBuilder currentBuilder; + if (builder != null) { + currentBuilder = builder.beginBlockEntry(); + } + else { + currentBuilder = type.createBlockBuilder(null, map.size()); + } + + for (Map.Entry entry : map.entrySet()) { + // Hive skips map entries with null keys + if (entry.getKey() != null) { + serializeObject(keyType, currentBuilder, entry.getKey()); + serializeObject(valueType, currentBuilder, entry.getValue()); + } + } + + if (builder != null) { + builder.closeEntry(); + return null; + } + else { + Block resultBlock = currentBuilder.build(); + return resultBlock; + } + } + + private static Block serializeStruct(Type type, BlockBuilder builder, Object object) + { + if (object == null) { + requireNonNull(builder, "parent builder is null").appendNull(); + return null; + } + + List typeParameters = type.getTypeParameters(); + ThriftGenericRow structData = (ThriftGenericRow) object; + BlockBuilder currentBuilder; + if (builder != null) { + currentBuilder = builder.beginBlockEntry(); + } + else { + currentBuilder = type.createBlockBuilder(null, typeParameters.size()); + } + + for (int i = 0; i < typeParameters.size(); i++) { + // TODO: Handle cases where ids are not consecutive + Object fieldValue = structData.getFieldValueForThriftId((short) (i + 1)); + serializeObject(typeParameters.get(i), currentBuilder, fieldValue); + } + + if (builder != null) { + builder.closeEntry(); + return null; + } + else { + Block resultBlock = currentBuilder.build(); + return resultBlock; + } + } + + private static void serializePrimitive(Type type, BlockBuilder builder, Object object) + { + requireNonNull(builder, "parent builder is null"); + + if (object == null) { + builder.appendNull(); + return; + } + + if (BOOLEAN.equals(type)) { + BOOLEAN.writeBoolean(builder, (Boolean) object); + } + else if (BIGINT.equals(type) || INTEGER.equals(type) || SMALLINT.equals(type) || TINYINT.equals(type) + || REAL.equals(type) || DATE.equals(type) || TIMESTAMP.equals(type)) { + type.writeLong(builder, getLongExpressedValue(object)); + } + else if (DOUBLE.equals(type)) { + DOUBLE.writeDouble(builder, ((Number) object).doubleValue()); + } + else if (isVarcharType(type) || VARBINARY.equals(type) || isCharType(type)) { + type.writeSlice(builder, getSliceExpressedValue(object, type)); + } + else { + throw new UnsupportedOperationException("Unsupported primitive type: " + type); + } + } + + public static boolean isArrayType(Type type) + { + return type.getTypeSignature().getBase().equals(StandardTypes.ARRAY); + } + + public static boolean isMapType(Type type) + { + return type.getTypeSignature().getBase().equals(StandardTypes.MAP); + } + + public static boolean isRowType(Type type) + { + return type.getTypeSignature().getBase().equals(StandardTypes.ROW); + } + + public static boolean isStructuralType(Type type) + { + String baseName = type.getTypeSignature().getBase(); + return baseName.equals(StandardTypes.MAP) || baseName.equals(StandardTypes.ARRAY) || baseName.equals(StandardTypes.ROW); + } + } +} diff --git a/presto-record-decoder/src/main/java/com/facebook/presto/decoder/thrift/ThriftGenericRow.java b/presto-record-decoder/src/main/java/com/facebook/presto/decoder/thrift/ThriftGenericRow.java new file mode 100644 index 0000000000000..036a7e60bbbf0 --- /dev/null +++ b/presto-record-decoder/src/main/java/com/facebook/presto/decoder/thrift/ThriftGenericRow.java @@ -0,0 +1,246 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.decoder.thrift; + +import org.apache.commons.lang.ArrayUtils; +import org.apache.thrift.TBase; +import org.apache.thrift.TException; +import org.apache.thrift.TFieldIdEnum; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TField; +import org.apache.thrift.protocol.TList; +import org.apache.thrift.protocol.TMap; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolUtil; +import org.apache.thrift.protocol.TSet; +import org.apache.thrift.protocol.TType; +import org.apache.thrift.transport.TMemoryInputTransport; +import org.apache.thrift.transport.TTransport; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class ThriftGenericRow + implements TBase +{ + private final Map values = new HashMap<>(); + private byte[] buf; + private int off; + private int len; + + public ThriftGenericRow() + { + } + + public ThriftGenericRow(Map values) + { + this.values.putAll(values); + } + + public class Fields + implements TFieldIdEnum + { + private final short thriftId; + private final String fieldName; + + Fields(short thriftId, String fieldName) + { + this.thriftId = thriftId; + this.fieldName = fieldName; + } + + public short getThriftFieldId() + { + return thriftId; + } + + public String getFieldName() + { + return fieldName; + } + } + + public void read(TProtocol iprot) + throws TException + { + TTransport trans = iprot.getTransport(); + buf = trans.getBuffer(); + off = trans.getBufferPosition(); + TProtocolUtil.skip(iprot, TType.STRUCT); + len = trans.getBufferPosition() - off; + } + + public void parse() + throws TException + { + parse(null); + } + + public void parse(short[] thriftIds) + throws TException + { + Set idSet = thriftIds == null ? null : new HashSet(Arrays.asList(ArrayUtils.toObject(thriftIds))); + TMemoryInputTransport trans = new TMemoryInputTransport(buf, off, len); + TBinaryProtocol iprot = new TBinaryProtocol(trans); + TField field; + iprot.readStructBegin(); + while (true) { + field = iprot.readFieldBegin(); + if (field.type == TType.STOP) { + break; + } + if (idSet != null && !idSet.remove(Short.valueOf(field.id))) { + TProtocolUtil.skip(iprot, field.type); + } + else { + values.put(field.id, readElem(iprot, field.type)); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + } + + private Object readElem(TProtocol iprot, byte type) + throws TException + { + switch (type) { + case TType.BOOL: + return iprot.readBool(); + case TType.BYTE: + return iprot.readByte(); + case TType.I16: + return iprot.readI16(); + case TType.ENUM: + case TType.I32: + return iprot.readI32(); + case TType.I64: + return iprot.readI64(); + case TType.DOUBLE: + return iprot.readDouble(); + case TType.STRING: + return iprot.readString(); + case TType.STRUCT: + return readStruct(iprot); + case TType.LIST: + return readList(iprot); + case TType.SET: + return readSet(iprot); + case TType.MAP: + return readMap(iprot); + default: + TProtocolUtil.skip(iprot, type); + return null; + } + } + + private Object readStruct(TProtocol iprot) + throws TException + { + ThriftGenericRow elem = new ThriftGenericRow(); + elem.read(iprot); + elem.parse(); + return elem; + } + + private Object readList(TProtocol iprot) + throws TException + { + TList ilist = iprot.readListBegin(); + List listValue = new ArrayList<>(); + for (int i = 0; i < ilist.size; i++) { + listValue.add(readElem(iprot, ilist.elemType)); + } + iprot.readListEnd(); + return listValue; + } + + private Object readSet(TProtocol iprot) + throws TException + { + TSet iset = iprot.readSetBegin(); + List setValue = new ArrayList<>(); + for (int i = 0; i < iset.size; i++) { + setValue.add(readElem(iprot, iset.elemType)); + } + iprot.readSetEnd(); + return setValue; + } + + private Object readMap(TProtocol iprot) + throws TException + { + TMap imap = iprot.readMapBegin(); + Map mapValue = new HashMap<>(); + for (int i = 0; i < imap.size; i++) { + mapValue.put(readElem(iprot, imap.keyType), readElem(iprot, imap.valueType)); + } + iprot.readMapEnd(); + return mapValue; + } + + public Object getFieldValueForThriftId(short thriftId) + { + return values.get(thriftId); + } + + public ThriftGenericRow deepCopy() + { + return new ThriftGenericRow(values); + } + + public void clear() + { + } + + public Fields fieldForId(int fieldId) + { + return new Fields((short) fieldId, "dummy"); + } + + public Object getFieldValue(Fields field) + { + return values.get(field.thriftId); + } + + public boolean isSet(Fields field) + { + return values.containsKey(field.getThriftFieldId()); + } + + public void setFieldValue(Fields field, Object value) + { + values.put(field.getThriftFieldId(), value); + } + + public void write(TProtocol oprot) + throws TException + { + throw new UnsupportedOperationException("ThriftGenericRow.write is not supported."); + } + + public Map getValues() + { + return values; + } + + public int compareTo(ThriftGenericRow other) + { + throw new UnsupportedOperationException("ThriftGenericRow.compareTo is not supported."); + } +} diff --git a/presto-record-decoder/src/main/java/com/facebook/presto/decoder/thrift/ThriftRowDecoder.java b/presto-record-decoder/src/main/java/com/facebook/presto/decoder/thrift/ThriftRowDecoder.java new file mode 100644 index 0000000000000..5816eb64b4283 --- /dev/null +++ b/presto-record-decoder/src/main/java/com/facebook/presto/decoder/thrift/ThriftRowDecoder.java @@ -0,0 +1,99 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.decoder.thrift; + +import com.facebook.presto.decoder.DecoderColumnHandle; +import com.facebook.presto.decoder.FieldValueProvider; +import com.facebook.presto.decoder.RowDecoder; +import com.google.common.base.Splitter; +import org.apache.thrift.TDeserializer; +import org.apache.thrift.TException; + +import java.util.Iterator; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static java.util.Objects.requireNonNull; +import static java.util.function.Function.identity; + +/** + * Thrift specific row decoder + */ +public class ThriftRowDecoder + implements RowDecoder +{ + public static final String NAME = "thrift"; + + private final Map columnDecoders; + + public ThriftRowDecoder(Set columnHandles) + { + requireNonNull(columnHandles, "columnHandles is null"); + columnDecoders = columnHandles.stream() + .collect(toImmutableMap(identity(), this::createColumnDecoder)); + } + + private ThriftColumnDecoder createColumnDecoder(DecoderColumnHandle columnHandle) + { + return new ThriftColumnDecoder(columnHandle); + } + + private static Object locateNode(Map map, DecoderColumnHandle columnHandle) + { + Map currentLevel = map; + Object val = null; + + Iterator it = Splitter.on('/').omitEmptyStrings().split(columnHandle.getMapping()).iterator(); + while (it.hasNext()) { + String pathElement = it.next(); + Short key = Short.valueOf(pathElement); + val = currentLevel.get(key); + + // could be because of optional fields + if (val == null) { + return null; + } + + if (val instanceof ThriftGenericRow) { + currentLevel = ((ThriftGenericRow) val).getValues(); + } + else if (it.hasNext()) { + throw new IllegalStateException("Invalid thrift field schema"); + } + } + + return val; + } + + @Override + public Optional> decodeRow(byte[] data, Map dataMap) + { + ThriftGenericRow row = new ThriftGenericRow(); + try { + TDeserializer deser = new TDeserializer(); + deser.deserialize(row, data); + row.parse(); + } + catch (TException e) { + return Optional.empty(); + } + + return Optional.of(columnDecoders.entrySet().stream() + .collect(toImmutableMap( + Map.Entry::getKey, + entry -> entry.getValue().decode(locateNode(row.getValues(), entry.getKey()))))); + } +} diff --git a/presto-record-decoder/src/main/java/com/facebook/presto/decoder/thrift/ThriftRowDecoderFactory.java b/presto-record-decoder/src/main/java/com/facebook/presto/decoder/thrift/ThriftRowDecoderFactory.java new file mode 100644 index 0000000000000..50c151578ec05 --- /dev/null +++ b/presto-record-decoder/src/main/java/com/facebook/presto/decoder/thrift/ThriftRowDecoderFactory.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.decoder.thrift; + +import com.facebook.presto.decoder.DecoderColumnHandle; +import com.facebook.presto.decoder.RowDecoder; +import com.facebook.presto.decoder.RowDecoderFactory; + +import java.util.Map; +import java.util.Set; + +public class ThriftRowDecoderFactory + implements RowDecoderFactory +{ + @Override + public RowDecoder create(Map decoderParams, Set columns) + { + return new ThriftRowDecoder(columns); + } +} diff --git a/presto-record-decoder/src/test/java/com/facebook/presto/decoder/thrift/TestThriftDecoder.java b/presto-record-decoder/src/test/java/com/facebook/presto/decoder/thrift/TestThriftDecoder.java new file mode 100644 index 0000000000000..aa57192c52d3e --- /dev/null +++ b/presto-record-decoder/src/test/java/com/facebook/presto/decoder/thrift/TestThriftDecoder.java @@ -0,0 +1,101 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.decoder.thrift; + +import com.facebook.presto.decoder.DecoderColumnHandle; +import com.facebook.presto.decoder.DecoderTestColumnHandle; +import com.facebook.presto.decoder.FieldValueProvider; +import com.facebook.presto.decoder.RowDecoder; +import com.facebook.presto.decoder.thrift.tweep.Location; +import com.facebook.presto.decoder.thrift.tweep.Tweet; +import com.facebook.presto.decoder.thrift.tweep.TweetType; +import com.facebook.presto.spi.type.BigintType; +import com.facebook.presto.spi.type.BooleanType; +import com.facebook.presto.spi.type.DoubleType; +import com.facebook.presto.spi.type.IntegerType; +import com.facebook.presto.spi.type.SmallintType; +import com.facebook.presto.spi.type.TinyintType; +import com.facebook.presto.spi.type.VarbinaryType; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.transport.TMemoryBuffer; +import org.testng.annotations.Test; + +import java.util.Map; +import java.util.Set; + +import static com.facebook.presto.decoder.util.DecoderTestUtil.checkValue; +import static com.facebook.presto.spi.type.VarcharType.createVarcharType; +import static java.util.Collections.emptyMap; +import static org.testng.Assert.assertEquals; + +public class TestThriftDecoder +{ + private static final ThriftRowDecoderFactory DECODER_FACTORY = new ThriftRowDecoderFactory(); + + @Test + public void testSimple() + throws Exception + { + Tweet tweet = new Tweet(1, "newUser", "hello world") + .setLoc(new Location(1234, 5678)) + .setAge((short) 26) + .setB((byte) 10) + .setIsDeleted(false) + .setTweetType(TweetType.REPLY) + .setFullId(1234567) + .setPic("abc".getBytes()) + .setAttr(ImmutableMap.of("a", "a")); + + // schema + DecoderTestColumnHandle col1 = new DecoderTestColumnHandle(1, "user_id", IntegerType.INTEGER, "1", "thrift", null, false, false, false); + DecoderTestColumnHandle col2 = new DecoderTestColumnHandle(2, "username", createVarcharType(100), "2", "thrift", null, false, false, false); + DecoderTestColumnHandle col3 = new DecoderTestColumnHandle(3, "text", createVarcharType(100), "3", "thrift", null, false, false, false); + DecoderTestColumnHandle col4 = new DecoderTestColumnHandle(4, "loc.latitude", DoubleType.DOUBLE, "4/1", "thrift", null, false, false, false); + DecoderTestColumnHandle col5 = new DecoderTestColumnHandle(5, "loc.longitude", DoubleType.DOUBLE, "4/2", "thrift", null, false, false, false); + DecoderTestColumnHandle col6 = new DecoderTestColumnHandle(6, "tweet_type", BigintType.BIGINT, "5", "thrift", null, false, false, false); + DecoderTestColumnHandle col7 = new DecoderTestColumnHandle(7, "is_deleted", BooleanType.BOOLEAN, "6", "thrift", null, false, false, false); + DecoderTestColumnHandle col8 = new DecoderTestColumnHandle(8, "b", TinyintType.TINYINT, "7", "thrift", null, false, false, false); + DecoderTestColumnHandle col9 = new DecoderTestColumnHandle(9, "age", SmallintType.SMALLINT, "8", "thrift", null, false, false, false); + DecoderTestColumnHandle col10 = new DecoderTestColumnHandle(10, "full_id", BigintType.BIGINT, "9", "thrift", null, false, false, false); + DecoderTestColumnHandle col11 = new DecoderTestColumnHandle(11, "pic", VarbinaryType.VARBINARY, "10", "thrift", null, false, false, false); + DecoderTestColumnHandle col12 = new DecoderTestColumnHandle(12, "language", createVarcharType(100), "16", "thrift", null, false, false, false); + + Set columns = ImmutableSet.of(col1, col2, col3, col4, col5, col6, col7, col8, col9, col10, col11, col12); + RowDecoder rowDecoder = DECODER_FACTORY.create(emptyMap(), columns); + + TMemoryBuffer transport = new TMemoryBuffer(4096); + TBinaryProtocol protocol = new TBinaryProtocol(transport); + tweet.write(protocol); + + Map decodedRow = rowDecoder.decodeRow(transport.getArray(), null) + .orElseThrow(AssertionError::new); + + assertEquals(decodedRow.size(), columns.size()); + + checkValue(decodedRow, col1, 1); + checkValue(decodedRow, col2, "newUser"); + checkValue(decodedRow, col3, "hello world"); + checkValue(decodedRow, col4, 1234); + checkValue(decodedRow, col5, 5678); + checkValue(decodedRow, col6, TweetType.REPLY.getValue()); + checkValue(decodedRow, col7, false); + checkValue(decodedRow, col8, 10); + checkValue(decodedRow, col9, 26); + checkValue(decodedRow, col10, 1234567); + checkValue(decodedRow, col11, "abc"); + checkValue(decodedRow, col12, "english"); + } +} diff --git a/presto-record-decoder/src/test/thrift/tweep.thrift b/presto-record-decoder/src/test/thrift/tweep.thrift new file mode 100644 index 0000000000000..57bcc7c317e74 --- /dev/null +++ b/presto-record-decoder/src/test/thrift/tweep.thrift @@ -0,0 +1,43 @@ +namespace java com.facebook.presto.decoder.thrift.tweep + +enum TweetType { + TWEET, + RETWEET = 2, + DM = 0xa, + REPLY +} + +struct Location { + 1: required double latitude; + 2: required double longitude; +} + +struct Tweet { + 1: required i32 userId; + 2: required string userName; + 3: required string text; + 4: optional Location loc; + 5: optional TweetType tweetType = TweetType.TWEET; + 6: optional bool isDeleted = false; + 7: optional byte b; + 8: optional i16 age; + 9: optional i64 fullId; + 10: optional binary pic; + 11: optional map attr; + 12: optional list items; + 16: optional string language = "english"; +} + +typedef list TweetList +typedef set TweetSet + +struct TweetSearchResult { + 1: TweetList tweetList; + 2: TweetSet tweetSet; +} + +exception TwitterUnavailable { + 1: string message; +} + +const i32 MAX_RESULTS = 100; diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index 6c67032c42c07..9229224b4cb86 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-redis diff --git a/presto-redshift/pom.xml b/presto-redshift/pom.xml index 40d504f9a6555..7cf35f27a7d6b 100644 --- a/presto-redshift/pom.xml +++ b/presto-redshift/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-redshift diff --git a/presto-resource-group-managers/pom.xml b/presto-resource-group-managers/pom.xml index 1811bca126484..b289a837ec37c 100644 --- a/presto-resource-group-managers/pom.xml +++ b/presto-resource-group-managers/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-resource-group-managers diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index a6f65391b2353..86dd879c44378 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index c8e57e3d83443..1c1755b13d0d8 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-server diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 51fc6a7910d33..b4c502de793ac 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-spi diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/Constraint.java b/presto-spi/src/main/java/com/facebook/presto/spi/Constraint.java index f42c503388357..4f7ceb7b7a0f1 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/Constraint.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/Constraint.java @@ -13,9 +13,11 @@ */ package com.facebook.presto.spi; +import com.facebook.presto.spi.predicate.FieldSet; import com.facebook.presto.spi.predicate.NullableValue; import com.facebook.presto.spi.predicate.TupleDomain; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.function.Predicate; @@ -26,6 +28,7 @@ public class Constraint { private final TupleDomain summary; private final Optional>> predicate; + private final Optional>> fieldSets; public static Constraint alwaysTrue() { @@ -42,18 +45,30 @@ public Constraint(TupleDomain summary) this(summary, Optional.empty()); } + public Constraint(Optional>> fieldSets) + { + this(TupleDomain.all(), Optional.empty(), fieldSets); + } + public Constraint(TupleDomain summary, Predicate> predicate) { this(summary, Optional.of(predicate)); } public Constraint(TupleDomain summary, Optional>> predicate) + { + this(summary, predicate, Optional.empty()); + } + + public Constraint(TupleDomain summary, Optional>> predicate, Optional>> fieldSets) { requireNonNull(summary, "summary is null"); requireNonNull(predicate, "predicate is null"); + requireNonNull(fieldSets, "fieldSets is null"); this.summary = summary; this.predicate = predicate; + this.fieldSets = fieldSets; } public TupleDomain getSummary() @@ -65,4 +80,9 @@ public Optional>> predicate() { return predicate; } + + public Optional>> fieldSets() + { + return fieldSets; + } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/predicate/FieldSet.java b/presto-spi/src/main/java/com/facebook/presto/spi/predicate/FieldSet.java new file mode 100644 index 0000000000000..1e61a2e40b239 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/predicate/FieldSet.java @@ -0,0 +1,54 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.predicate; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +public class FieldSet +{ + private final T column; + private final Set fields; + + @JsonCreator + public FieldSet( + @JsonProperty("column") T column, + @JsonProperty("fields") Set fields) + { + this.column = requireNonNull(column, "column is null"); + this.fields = requireNonNull(fields, "fields is null"); + } + + @JsonProperty + public T getColumn() + { + return column; + } + + @JsonProperty + public Set getFields() + { + return fields; + } + + @Override + public String toString() + { + return "[" + column.toString() + " : " + fields.toString() + "]"; + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeSignature.java b/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeSignature.java index 72519acc07354..d584b77912868 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeSignature.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/type/TypeSignature.java @@ -47,7 +47,7 @@ public class TypeSignature static { BASE_NAME_ALIAS_TO_CANONICAL.put("int", StandardTypes.INTEGER); - + BASE_NAME_ALIAS_TO_CANONICAL.put("", "unknown"); SIMPLE_TYPE_WITH_SPACES.add(StandardTypes.TIME_WITH_TIME_ZONE); SIMPLE_TYPE_WITH_SPACES.add(StandardTypes.TIMESTAMP_WITH_TIME_ZONE); SIMPLE_TYPE_WITH_SPACES.add(StandardTypes.INTERVAL_DAY_TO_SECOND); diff --git a/presto-spi/src/test/java/com/facebook/presto/spi/type/TestTypeSignature.java b/presto-spi/src/test/java/com/facebook/presto/spi/type/TestTypeSignature.java index 66805283c9878..994583ea89556 100644 --- a/presto-spi/src/test/java/com/facebook/presto/spi/type/TestTypeSignature.java +++ b/presto-spi/src/test/java/com/facebook/presto/spi/type/TestTypeSignature.java @@ -248,9 +248,9 @@ public void parseSignature() "map", ImmutableList.of("bigint", "map(bigint,map(varchar,bigint))")); - assertSignatureFail("blah()"); - assertSignatureFail("array()"); - assertSignatureFail("map()"); + assertSignature("blah()", "blah", ImmutableList.of("unknown"), "blah(unknown)"); + assertSignature("array()", "array", ImmutableList.of("unknown"), "array(unknown)"); + assertSignature("map()", "map", ImmutableList.of("unknown"), "map(unknown)"); assertSignatureFail("x", ImmutableSet.of("x")); // ensure this is not treated as a row type diff --git a/presto-sqlserver/pom.xml b/presto-sqlserver/pom.xml index b7a01729ecee4..4f3318e112fa3 100644 --- a/presto-sqlserver/pom.xml +++ b/presto-sqlserver/pom.xml @@ -3,7 +3,7 @@ presto-root com.facebook.presto - 0.208 + 0.208-tw-0.53 4.0.0 diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index b5da6617b75da..c6e9c285ad54b 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 289898a63d03b..d2b62dda1bc35 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-testing-server-launcher diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 1208a06d2d303..ff965e906b364 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ presto-root com.facebook.presto - 0.208 + 0.208-tw-0.53 presto-tests diff --git a/presto-thrift-connector-api/pom.xml b/presto-thrift-connector-api/pom.xml index 9617e782eb56b..b5f6dc554b88b 100644 --- a/presto-thrift-connector-api/pom.xml +++ b/presto-thrift-connector-api/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-thrift-connector-api diff --git a/presto-thrift-connector/pom.xml b/presto-thrift-connector/pom.xml index 4eca220613ce0..d6910d0c8fe73 100644 --- a/presto-thrift-connector/pom.xml +++ b/presto-thrift-connector/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-thrift-connector diff --git a/presto-thrift-testing-server/pom.xml b/presto-thrift-testing-server/pom.xml index 866491aa426dd..fe3074a5fdb44 100644 --- a/presto-thrift-testing-server/pom.xml +++ b/presto-thrift-testing-server/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-thrift-testing-server diff --git a/presto-thrift-testing-server/src/main/java/com/facebook/presto/connector/thrift/server/ThriftIndexedTpchService.java b/presto-thrift-testing-server/src/main/java/com/facebook/presto/connector/thrift/server/ThriftIndexedTpchService.java index bd075b9690ae9..3ce8a4a2edd90 100644 --- a/presto-thrift-testing-server/src/main/java/com/facebook/presto/connector/thrift/server/ThriftIndexedTpchService.java +++ b/presto-thrift-testing-server/src/main/java/com/facebook/presto/connector/thrift/server/ThriftIndexedTpchService.java @@ -96,8 +96,12 @@ protected ConnectorPageSource createLookupPageSource(SplitInfo splitInfo, List new IllegalArgumentException(String.format("No such index: %s%s", splitInfo.getTableName(), splitInfo.getLookupColumnNames()))); - List lookupColumnTypes = types(splitInfo.getTableName(), splitInfo.getLookupColumnNames()); - RecordSet keyRecordSet = new ListBasedRecordSet(splitInfo.getKeys(), lookupColumnTypes); + ImmutableList.Builder> keyBuilder = ImmutableList.builder(); + for (String column : indexedTable.getKeyColumns()) { + keyBuilder.add(splitInfo.getKeys().get(splitInfo.getLookupColumnNames().indexOf(column))); + } + List lookupColumnTypes = types(splitInfo.getTableName(), indexedTable.getKeyColumns()); + RecordSet keyRecordSet = new ListBasedRecordSet(keyBuilder.build(), lookupColumnTypes); RecordSet outputRecordSet = lookupIndexKeys(keyRecordSet, indexedTable, outputColumnNames); return new RecordPageSource(outputRecordSet); } diff --git a/presto-tpcds/pom.xml b/presto-tpcds/pom.xml index ab5ee34438018..2ccaa4d15b8ad 100644 --- a/presto-tpcds/pom.xml +++ b/presto-tpcds/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-tpcds diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index f7c07e9392b50..533b3c6b1b158 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-tpch diff --git a/presto-twitter-functions/pom.xml b/presto-twitter-functions/pom.xml new file mode 100644 index 0000000000000..38322e3cbcd46 --- /dev/null +++ b/presto-twitter-functions/pom.xml @@ -0,0 +1,65 @@ + + + 4.0.0 + + + com.facebook.presto + presto-root + 0.208-tw-0.53 + + + presto-twitter-functions + Twitter's specific functions for Presto + presto-plugin + + + ${project.parent.basedir} + + + + + com.google.guava + guava + + + + + com.facebook.presto + presto-spi + provided + + + + io.airlift + slice + provided + + + + + org.testng + testng + test + + + + com.facebook.presto + presto-tests + test + + + + com.facebook.presto + presto-main + test + + + + com.facebook.presto + presto-main + test-jar + test + + + + diff --git a/presto-twitter-functions/src/main/java/com/facebook/presto/twitter/functions/SnowflakeFunctions.java b/presto-twitter-functions/src/main/java/com/facebook/presto/twitter/functions/SnowflakeFunctions.java new file mode 100644 index 0000000000000..979cb95a8504d --- /dev/null +++ b/presto-twitter-functions/src/main/java/com/facebook/presto/twitter/functions/SnowflakeFunctions.java @@ -0,0 +1,129 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.functions; + +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.function.Description; +import com.facebook.presto.spi.function.ScalarFunction; +import com.facebook.presto.spi.function.SqlType; +import com.facebook.presto.spi.type.StandardTypes; + +import java.util.concurrent.TimeUnit; + +import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; + +/* size in + * bits: 1 41 5 5 12 + * +-+-----------------------------------------+-----+-----+------------+ + * |0| milliseconds |clstr|instc| sequence | + * |0| since twepoch | id | id | number | + * +-+-----------------------------------------+-----+-----+------------+ + * | + * +- Most significant bit + */ +public class SnowflakeFunctions +{ + private static final long SequenceNumBits = 12L; + private static final long MaxSequenceNum = (1L << SequenceNumBits) - 1; + private static final long SequenceNumMask = MaxSequenceNum; + + private static final long InstanceIdBits = 5L; + private static final long MaxInstanceId = (1L << InstanceIdBits) - 1; + private static final long InstanceIdShift = SequenceNumBits; + private static final long InstanceIdMask = MaxInstanceId << InstanceIdShift; + + private static final long ClusterIdBits = 5L; + private static final long MaxClusterId = (1L << ClusterIdBits) - 1; + private static final long ClusterIdShift = InstanceIdShift + InstanceIdBits; + private static final long ClusterIdMask = MaxClusterId << ClusterIdShift; + + private static final long TimestampBits = 41L; + private static final long MaxTimestamp = (1L << TimestampBits) - 1; + private static final long TimestampShift = ClusterIdShift + ClusterIdBits; + private static final long TimestampMask = MaxTimestamp << TimestampShift; + + /* Twepoch is 2010-11-04T01:42:54Z. + * Value is in millis since Unix Epoch 1970-01-01T00:00:00Z. + */ + private static final long Twepoch = 1288834974657L; + private static final long FirstSnowflakeIdUnixTime = Twepoch + TimeUnit.DAYS.toMillis(1); // 1 day after Twepoch. + private static final long FirstSnowflakeId = firstSnowflakeIdFor(FirstSnowflakeIdUnixTime); + + private SnowflakeFunctions() + { + } + + @ScalarFunction("is_snowflake") + @Description("Check if a BIGINT is a Snowflake ID") + @SqlType(StandardTypes.BOOLEAN) + public static boolean isSnowflakeId(@SqlType(StandardTypes.BIGINT) long id) + { + return id >= FirstSnowflakeId; + } + + @ScalarFunction("first_snowflake_for") + @Description("Return the first snowflake ID given a timestamp") + @SqlType(StandardTypes.BIGINT) + public static long firstSnowflakeIdFor(@SqlType(StandardTypes.TIMESTAMP) long timestamp) + { + if (timestamp < FirstSnowflakeIdUnixTime) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Invalid UnixTimeMillis: UnixTimeMillis[" + timestamp + "] >= FirstSnowflakeIdUnixTime"); + } + return ((timestamp - Twepoch) << TimestampShift); + } + + @ScalarFunction("timestamp_from_snowflake") + @Description("Return the timestamp given a snowflake ID") + @SqlType(StandardTypes.TIMESTAMP) + public static long timestampFromSnowflakeId(@SqlType(StandardTypes.BIGINT) long id) + { + if (!isSnowflakeId(id)) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Not a Snowflake Id: " + id); + } + return ((id & TimestampMask) >> TimestampShift) + Twepoch; + } + + @ScalarFunction("cluster_id_from_snowflake") + @Description("Return the cluster id given a snowflake ID") + @SqlType(StandardTypes.BIGINT) + public static long clusterIdFromSnowflakeId(@SqlType(StandardTypes.BIGINT) long id) + { + if (!isSnowflakeId(id)) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Not a Snowflake Id: " + id); + } + return (id & ClusterIdMask) >> ClusterIdShift; + } + + @ScalarFunction("instance_id_from_snowflake") + @Description("Return the instance id given a snowflake ID") + @SqlType(StandardTypes.BIGINT) + public static long instanceIdFromSnowflakeId(@SqlType(StandardTypes.BIGINT) long id) + { + if (!isSnowflakeId(id)) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Not a Snowflake Id: " + id); + } + return (id & InstanceIdMask) >> InstanceIdShift; + } + + @ScalarFunction("sequence_num_from_snowflake") + @Description("Return the sequence number given a snowflake ID") + @SqlType(StandardTypes.BIGINT) + public static long sequenceNumFromSnowflakeId(@SqlType(StandardTypes.BIGINT) long id) + { + if (!isSnowflakeId(id)) { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Not a Snowflake Id: " + id); + } + return id & SequenceNumMask; + } +} diff --git a/presto-twitter-functions/src/main/java/com/facebook/presto/twitter/functions/TwitterFunctionsPlugin.java b/presto-twitter-functions/src/main/java/com/facebook/presto/twitter/functions/TwitterFunctionsPlugin.java new file mode 100644 index 0000000000000..6a9a491e9306d --- /dev/null +++ b/presto-twitter-functions/src/main/java/com/facebook/presto/twitter/functions/TwitterFunctionsPlugin.java @@ -0,0 +1,32 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.functions; + +import com.facebook.presto.spi.Plugin; +import com.google.common.collect.ImmutableSet; + +import java.util.Set; + +public class TwitterFunctionsPlugin + implements Plugin +{ + @Override + public Set> getFunctions() + { + return ImmutableSet.>builder() + .add(TwitterStringFunctions.class) + .add(SnowflakeFunctions.class) + .build(); + } +} diff --git a/presto-twitter-functions/src/main/java/com/facebook/presto/twitter/functions/TwitterStringFunctions.java b/presto-twitter-functions/src/main/java/com/facebook/presto/twitter/functions/TwitterStringFunctions.java new file mode 100644 index 0000000000000..9f43baaafe3a9 --- /dev/null +++ b/presto-twitter-functions/src/main/java/com/facebook/presto/twitter/functions/TwitterStringFunctions.java @@ -0,0 +1,102 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.functions; + +import com.facebook.presto.spi.ErrorCodeSupplier; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.function.Description; +import com.facebook.presto.spi.function.LiteralParameters; +import com.facebook.presto.spi.function.ScalarFunction; +import com.facebook.presto.spi.function.SqlType; +import com.facebook.presto.spi.type.StandardTypes; +import com.google.common.primitives.Ints; +import io.airlift.slice.Slice; + +import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; +import static com.facebook.presto.spi.type.VarcharType.VARCHAR; +import static io.airlift.slice.SliceUtf8.offsetOfCodePoint; +import static java.lang.String.format; + +public class TwitterStringFunctions +{ + private TwitterStringFunctions() + { + } + + @ScalarFunction("split_every") + @Description("Splits the string on every character and returns an array") + @LiteralParameters({"x"}) + @SqlType("array(varchar(x))") + public static Block str2array(@SqlType("varchar(x)") Slice utf8) + { + return str2array(utf8, 1, utf8.length() + 1); + } + + @ScalarFunction("split_every") + @Description("Splits the string on every given length of characters and returns an array") + @LiteralParameters({"x"}) + @SqlType("array(varchar(x))") + public static Block str2array(@SqlType("varchar(x)") Slice utf8, @SqlType(StandardTypes.BIGINT) long length) + { + return str2array(utf8, length, utf8.length() / length + 1); + } + + @ScalarFunction("split_every") + @Description("Splits the string on every given length of characters and returns an array with the size at most of the given limit") + @LiteralParameters({"x"}) + @SqlType("array(varchar(x))") + public static Block str2array(@SqlType("varchar(x)") Slice utf8, @SqlType(StandardTypes.BIGINT) long length, @SqlType(StandardTypes.BIGINT) long limit) + { + checkCondition(limit > 0, INVALID_FUNCTION_ARGUMENT, "Limit must be positive"); + checkCondition(limit <= Integer.MAX_VALUE, INVALID_FUNCTION_ARGUMENT, "Limit is too large"); + checkCondition(length > 0, INVALID_FUNCTION_ARGUMENT, "Length must be positive"); + checkCondition(length <= Integer.MAX_VALUE, INVALID_FUNCTION_ARGUMENT, "Length is too large"); + BlockBuilder parts = VARCHAR.createBlockBuilder(null, 1, Ints.saturatedCast(length)); + // If limit is one, the last and only element is the complete string + if (limit == 1) { + VARCHAR.writeSlice(parts, utf8); + return parts.build(); + } + + int index = offsetOfCodePoint(utf8, 0); + while (index < utf8.length()) { + int splitIndex = offsetOfCodePoint(utf8, index, Ints.saturatedCast(length)); + // Enough remaining string? + if (splitIndex < 0) { + break; + } + // Add the part from current index to found split + VARCHAR.writeSlice(parts, utf8, index, splitIndex - index); + // Continue after current end + index = splitIndex; + // Reached limit-1 parts so we can stop + if (parts.getPositionCount() == limit - 1) { + break; + } + } + // Rest of string + VARCHAR.writeSlice(parts, utf8, index, utf8.length() - index); + + return parts.build(); + } + + private static void checkCondition(boolean condition, ErrorCodeSupplier errorCode, String formatString, Object... args) + { + if (!condition) { + throw new PrestoException(errorCode, format(formatString, args)); + } + } +} diff --git a/presto-twitter-functions/src/test/java/com/facebook/presto/twitter/functions/TestTwitterFunctions.java b/presto-twitter-functions/src/test/java/com/facebook/presto/twitter/functions/TestTwitterFunctions.java new file mode 100644 index 0000000000000..4c75f59a88f54 --- /dev/null +++ b/presto-twitter-functions/src/test/java/com/facebook/presto/twitter/functions/TestTwitterFunctions.java @@ -0,0 +1,82 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.twitter.functions; + +import com.facebook.presto.operator.scalar.AbstractTestFunctions; +import com.facebook.presto.spi.type.ArrayType; +import com.facebook.presto.spi.type.SqlTimestamp; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static com.facebook.presto.metadata.FunctionExtractor.extractFunctions; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.TimeZoneKey.UTC_KEY; +import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; +import static com.facebook.presto.spi.type.VarcharType.createVarcharType; + +public class TestTwitterFunctions + extends AbstractTestFunctions +{ + @BeforeClass + public void setUp() + { + functionAssertions.addFunctions(extractFunctions(new TwitterFunctionsPlugin().getFunctions())); + } + + @Test + public void testStr2Array() + { + assertFunction("SPLIT_EVERY('')", new ArrayType(createVarcharType(0)), ImmutableList.of("")); + assertFunction("SPLIT_EVERY('abc')", new ArrayType(createVarcharType(3)), ImmutableList.of("a", "b", "c")); + assertFunction("SPLIT_EVERY('a.b.c')", new ArrayType(createVarcharType(5)), ImmutableList.of("a", ".", "b", ".", "c")); + assertFunction("SPLIT_EVERY('...')", new ArrayType(createVarcharType(3)), ImmutableList.of(".", ".", ".")); + // Test str_to_array for non-ASCII + assertFunction("SPLIT_EVERY('\u4FE1\u5FF5,\u7231,\u5E0C\u671B')", new ArrayType(createVarcharType(7)), ImmutableList.of("\u4FE1", "\u5FF5", ",", "\u7231", ",", "\u5E0C", "\u671B")); + // Test argument length + assertFunction("SPLIT_EVERY('a.b.c', 2)", new ArrayType(createVarcharType(5)), ImmutableList.of("a.", "b.", "c")); + // Test argument limit + assertFunction("SPLIT_EVERY('a.b.c', 2, 1)", new ArrayType(createVarcharType(5)), ImmutableList.of("a.b.c")); + assertFunction("SPLIT_EVERY('a.b.c', 2, 2)", new ArrayType(createVarcharType(5)), ImmutableList.of("a.", "b.c")); + } + + private static SqlTimestamp toTimestampUTC(long millis) + { + return new SqlTimestamp(millis, UTC_KEY); + } + + @Test + public void testSnowflake() + { + assertFunction("IS_SNOWFLAKE(1000)", BOOLEAN, false); + assertFunction("IS_SNOWFLAKE(265605588183052288)", BOOLEAN, true); + assertFunction("IS_SNOWFLAKE(-265605588183052288)", BOOLEAN, false); + + assertFunction("FIRST_SNOWFLAKE_FOR(from_unixtime(1352160281.593))", BIGINT, 265605588182892544L); + assertInvalidFunction("FIRST_SNOWFLAKE_FOR(from_unixtime(1000))", "Invalid UnixTimeMillis: UnixTimeMillis[1000000] >= FirstSnowflakeIdUnixTime"); + + assertFunction("TIMESTAMP_FROM_SNOWFLAKE(265605588183052288)", TIMESTAMP, toTimestampUTC(1352160281593L)); + assertInvalidFunction("TIMESTAMP_FROM_SNOWFLAKE(1000)", "Not a Snowflake Id: 1000"); + + assertFunction("CLUSTER_ID_FROM_SNOWFLAKE(265605588183052288)", BIGINT, 1L); + assertInvalidFunction("CLUSTER_ID_FROM_SNOWFLAKE(1000)", "Not a Snowflake Id: 1000"); + + assertFunction("INSTANCE_ID_FROM_SNOWFLAKE(265605588183052288)", BIGINT, 7L); + assertInvalidFunction("INSTANCE_ID_FROM_SNOWFLAKE(1000)", "Not a Snowflake Id: 1000"); + + assertFunction("SEQUENCE_NUM_FROM_SNOWFLAKE(265605588183052288)", BIGINT, 0L); + assertInvalidFunction("SEQUENCE_NUM_FROM_SNOWFLAKE(1000)", "Not a Snowflake Id: 1000"); + } +} diff --git a/presto-twitter-server/NOTICE b/presto-twitter-server/NOTICE new file mode 100644 index 0000000000000..6182bbb1148d3 --- /dev/null +++ b/presto-twitter-server/NOTICE @@ -0,0 +1,2836 @@ +THE FOLLOWING IS SOFTWARE LICENSED BY THIRD PARTIES UNDER OPEN SOURCE LICENSES THAT MAY BE USED BY THIS PRODUCT. + +----- + +The following software may be included in this product: aether. The source code is available at http://eclipse.org/aether/download/. You may also request a copy of the source code by sending a request to opensource@fb.com. This software contains the following license and notice below: + +Eclipse Public License - v 1.0 + +THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + +a) in the case of the initial Contributor, the initial code and documentation distributed under this Agreement, and + +b) in the case of each subsequent Contributor: + +i) changes to the Program, and + +ii) additions to the Program; + +where such changes and/or additions to the Program originate from and are distributed by that particular Contributor. A Contribution 'originates' from a Contributor if it was added to the Program by such Contributor itself or anyone acting on such Contributor's behalf. Contributions do not include additions to the Program which: (i) are separate modules of software distributed in conjunction with the Program under their own license agreement, and (ii) are not derivative works of the Program. + +"Contributor" means any person or entity that distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor which are necessarily infringed by the use or sale of its Contribution alone or when combined with the Program. + +"Program" means the Contributions distributed in accordance with this Agreement. + +"Recipient" means anyone who receives the Program under this Agreement, including all Contributors. + +2. GRANT OF RIGHTS + +a) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, distribute and sublicense the Contribution of such Contributor, if any, and such derivative works, in source code and object code form. + +b) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed Patents to make, use, sell, offer to sell, import and otherwise transfer the Contribution of such Contributor, if any, in source code and object code form. This patent license shall apply to the combination of the Contribution and the Program if, at the time the Contribution is added by the Contributor, such addition of the Contribution causes such combination to be covered by the Licensed Patents. The patent license shall not apply to any other combinations which include the Contribution. No hardware per se is licensed hereunder. + +c) Recipient understands that although each Contributor grants the licenses to its Contributions set forth herein, no assurances are provided by any Contributor that the Program does not infringe the patent or other intellectual property rights of any other entity. Each Contributor disclaims any liability to Recipient for claims brought by any other entity based on infringement of intellectual property rights or otherwise. As a condition to exercising the rights and licenses granted hereunder, each Recipient hereby assumes sole responsibility to secure any other intellectual property rights needed, if any. For example, if a third party patent license is required to allow Recipient to distribute the Program, it is Recipient's responsibility to acquire that license before distributing the Program. + +d) Each Contributor represents that to its knowledge it has sufficient copyright rights in its Contribution, if any, to grant the copyright license set forth in this Agreement. + +3. REQUIREMENTS + +A Contributor may choose to distribute the Program in object code form under its own license agreement, provided that: + +a) it complies with the terms and conditions of this Agreement; and + +b) its license agreement: + +i) effectively disclaims on behalf of all Contributors all warranties and conditions, express and implied, including warranties or conditions of title and non-infringement, and implied warranties or conditions of merchantability and fitness for a particular purpose; + +ii) effectively excludes on behalf of all Contributors all liability for damages, including direct, indirect, special, incidental and consequential damages, such as lost profits; + +iii) states that any provisions which differ from this Agreement are offered by that Contributor alone and not by any other party; and + +iv) states that source code for the Program is available from such Contributor, and informs licensees how to obtain it in a reasonable manner on or through a medium customarily used for software exchange. + +When the Program is made available in source code form: + +a) it must be made available under this Agreement; and + +b) a copy of this Agreement must be included with each copy of the Program. + +Contributors may not remove or alter any copyright notices contained within the Program. + +Each Contributor must identify itself as the originator of its Contribution, if any, in a manner that reasonably allows subsequent Recipients to identify the originator of the Contribution. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities with respect to end users, business partners and the like. While this license is intended to facilitate the commercial use of the Program, the Contributor who includes the Program in a commercial product offering should do so in a manner which does not create potential liability for other Contributors. Therefore, if a Contributor includes the Program in a commercial product offering, such Contributor ("Commercial Contributor") hereby agrees to defend and indemnify every other Contributor ("Indemnified Contributor") against any losses, damages and costs (collectively "Losses") arising from claims, lawsuits and other legal actions brought by a third party against the Indemnified Contributor to the extent caused by the acts or omissions of such Commercial Contributor in connection with its distribution of the Program in a commercial product offering. The obligations in this section do not apply to any claims or Losses relating to any actual or alleged intellectual property infringement. In order to qualify, an Indemnified Contributor must: a) promptly notify the Commercial Contributor in writing of such claim, and b) allow the Commercial Contributor to control, and cooperate with the Commercial Contributor in, the defense and any related settlement negotiations. The Indemnified Contributor may participate in any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial product offering, Product X. That Contributor is then a Commercial Contributor. If that Commercial Contributor then makes performance claims, or offers warranties related to Product X, those performance claims and warranties are such Commercial Contributor's responsibility alone. Under this section, the Commercial Contributor would have to defend claims against the other Contributors related to those performance claims and warranties, and if a court requires any other Contributor to pay any damages as a result, the Commercial Contributor must pay those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the appropriateness of using and distributing the Program and assumes all risks associated with its exercise of rights under this Agreement , including but not limited to the risks and costs of program errors, compliance with applicable laws, damage to or loss of data, programs or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this Agreement, and without further action by the parties hereto, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable. + +If Recipient institutes patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Program itself (excluding combinations of the Program with other software or hardware) infringes such Recipient's patent(s), then such Recipient's rights granted under Section 2(b) shall terminate as of the date such litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it fails to comply with any of the material terms or conditions of this Agreement and does not cure such failure in a reasonable period of time after becoming aware of such noncompliance. If all Recipient's rights under this Agreement terminate, Recipient agrees to cease use and distribution of the Program as soon as reasonably practicable. However, Recipient's obligations under this Agreement and any licenses granted by Recipient relating to the Program shall continue and survive. + +Everyone is permitted to copy and distribute copies of this Agreement, but in order to avoid inconsistency the Agreement is copyrighted and may only be modified in the following manner. The Agreement Steward reserves the right to publish new versions (including revisions) of this Agreement from time to time. No one other than the Agreement Steward has the right to modify this Agreement. The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation may assign the responsibility to serve as the Agreement Steward to a suitable separate entity. Each new version of the Agreement will be given a distinguishing version number. The Program (including Contributions) may always be distributed subject to the version of the Agreement under which it was received. In addition, after a new version of the Agreement is published, Contributor may elect to distribute the Program (including its Contributions) under the new version. Except as expressly stated in Sections 2(a) and 2(b) above, Recipient receives no rights or licenses to the intellectual property of any Contributor under this Agreement, whether expressly, by implication, estoppel or otherwise. All rights in the Program not expressly granted under this Agreement are reserved. + +This Agreement is governed by the laws of the State of New York and the intellectual property laws of the United States of America. No party to this Agreement will bring a legal action under this Agreement more than one year after the cause of action arose. Each party waives its rights to a jury trial in any resulting litigation. + +----- + +The following software may be included in this product: antlr stringtemplate4, antlr runtime. This software contains the following license and notice below: + +[The "BSD license"] +Copyright (c) 2011-2013 Terence Parr +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +----- + +The following software may be included in this product: Apache Avro, Apache BVal, Apache Commons BeanUtils Core, Apache Commons CLI, Apache Commons Codec, Apache Commons Configuration, Apache Commons IO, Apache Commons Lang, Apache Commons Logging, Apache Hadoop, Apache Hive, Apache HttpClient, Apache Maven, Apache Thrift, Apache XBean, Bean Validation API, Code Generation Library, Guava, Jackson, Jetty, Joda time, Log4j Implemented Over SLF4J, Ning Asynchronous Http Client, Plexus, Tableau Web Data Connector, airlift, airlift resolver, airlift slice, fastutil, jDBI, javax.inject, jmxutils, jQuery, opencsv, snappy, vis.js. +This software contains the following license and notice below: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +----- + +The following software may be included in this product: asm. This software contains the following license and notice below: + +Copyright (c) 2000-2011 INRIA, France Telecom +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. + +----- + +The following software may be included in this product: findbugs. The source code is available at http://code.google.com/p/findbugs/. You may also request a copy of the source code by sending a request to opensource@fb.com. This software contains the following license and notice below: + + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! + +-- From LICENSE-ASM.txt: + +Copyright (c) 2000-2005 INRIA, France Telecom +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holders nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. + +-- From LICENSE-AppleJavaExtensions.txt: + +AppleJavaExtensions +v 1.2 + +This is a pluggable jar of stub classes representing the new Apple eAWT and eIO APIs for Java 1.4 on Mac OS X. The purpose of these stubs is to allow for compilation of eAWT- or eIO-referencing code on platforms other than Mac OS X. The jar file is enclosed in a zip archive for easy expansion on other platforms. + +These stubs are not intended for the runtime classpath on non-Mac platforms. Please see the OSXAdapter sample for how to write cross-platform code that uses eAWT. + +Disclaimer: IMPORTANT: This Apple software is supplied to you by Apple +Computer, Inc. ("Apple") in consideration of your agreement to the +following terms, and your use, installation, modification or +redistribution of this Apple software constitutes acceptance of these +terms. If you do not agree with these terms, please do not use, +install, modify or redistribute this Apple software. + +In consideration of your agreement to abide by the following terms, and +subject to these terms, Apple grants you a personal, non-exclusive +license, under Apple's copyrights in this original Apple software (the +"Apple Software"), to use, reproduce, modify and redistribute the Apple +Software, with or without modifications, in source and/or binary forms; +provided that if you redistribute the Apple Software in its entirety and +without modifications, you must retain this notice and the following +text and disclaimers in all such redistributions of the Apple Software. +Neither the name, trademarks, service marks or logos of Apple Computer, +Inc. may be used to endorse or promote products derived from the Apple +Software without specific prior written permission from Apple. Except +as expressly stated in this notice, no other rights or licenses, express +or implied, are granted by Apple herein, including but not limited to +any patent rights that may be infringed by your derivative works or by +other works in which the Apple Software may be incorporated. + +The Apple Software is provided by Apple on an "AS IS" basis. APPLE +MAKES NO WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION +THE IMPLIED WARRANTIES OF NON-INFRINGEMENT, MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE, REGARDING THE APPLE SOFTWARE OR ITS USE AND +OPERATION ALONE OR IN COMBINATION WITH YOUR PRODUCTS. + +IN NO EVENT SHALL APPLE BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL +OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) ARISING IN ANY WAY OUT OF THE USE, REPRODUCTION, +MODIFICATION AND/OR DISTRIBUTION OF THE APPLE SOFTWARE, HOWEVER CAUSED +AND WHETHER UNDER THEORY OF CONTRACT, TORT (INCLUDING NEGLIGENCE), +STRICT LIABILITY OR OTHERWISE, EVEN IF APPLE HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +Copyright © 2003-2006 Apple Computer, Inc., All Rights Reserved + +-- From LICENSE-bcel.txt: + +/* + * Apache License + * Version 2.0, January 2004 + * http://www.apache.org/licenses/ + * + * TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + * + * 1. Definitions. + * + * "License" shall mean the terms and conditions for use, reproduction, + * and distribution as defined by Sections 1 through 9 of this document. + * + * "Licensor" shall mean the copyright owner or entity authorized by + * the copyright owner that is granting the License. + * + * "Legal Entity" shall mean the union of the acting entity and all + * other entities that control, are controlled by, or are under common + * control with that entity. For the purposes of this definition, + * "control" means (i) the power, direct or indirect, to cause the + * direction or management of such entity, whether by contract or + * otherwise, or (ii) ownership of fifty percent (50%) or more of the + * outstanding shares, or (iii) beneficial ownership of such entity. + * + * "You" (or "Your") shall mean an individual or Legal Entity + * exercising permissions granted by this License. + * + * "Source" form shall mean the preferred form for making modifications, + * including but not limited to software source code, documentation + * source, and configuration files. + * + * "Object" form shall mean any form resulting from mechanical + * transformation or translation of a Source form, including but + * not limited to compiled object code, generated documentation, + * and conversions to other media types. + * + * "Work" shall mean the work of authorship, whether in Source or + * Object form, made available under the License, as indicated by a + * copyright notice that is included in or attached to the work + * (an example is provided in the Appendix below). + * + * "Derivative Works" shall mean any work, whether in Source or Object + * form, that is based on (or derived from) the Work and for which the + * editorial revisions, annotations, elaborations, or other modifications + * represent, as a whole, an original work of authorship. For the purposes + * of this License, Derivative Works shall not include works that remain + * separable from, or merely link (or bind by name) to the interfaces of, + * the Work and Derivative Works thereof. + * + * "Contribution" shall mean any work of authorship, including + * the original version of the Work and any modifications or additions + * to that Work or Derivative Works thereof, that is intentionally + * submitted to Licensor for inclusion in the Work by the copyright owner + * or by an individual or Legal Entity authorized to submit on behalf of + * the copyright owner. For the purposes of this definition, "submitted" + * means any form of electronic, verbal, or written communication sent + * to the Licensor or its representatives, including but not limited to + * communication on electronic mailing lists, source code control systems, + * and issue tracking systems that are managed by, or on behalf of, the + * Licensor for the purpose of discussing and improving the Work, but + * excluding communication that is conspicuously marked or otherwise + * designated in writing by the copyright owner as "Not a Contribution." + * + * "Contributor" shall mean Licensor and any individual or Legal Entity + * on behalf of whom a Contribution has been received by Licensor and + * subsequently incorporated within the Work. + * + * 2. Grant of Copyright License. Subject to the terms and conditions of + * this License, each Contributor hereby grants to You a perpetual, + * worldwide, non-exclusive, no-charge, royalty-free, irrevocable + * copyright license to reproduce, prepare Derivative Works of, + * publicly display, publicly perform, sublicense, and distribute the + * Work and such Derivative Works in Source or Object form. + * + * 3. Grant of Patent License. Subject to the terms and conditions of + * this License, each Contributor hereby grants to You a perpetual, + * worldwide, non-exclusive, no-charge, royalty-free, irrevocable + * (except as stated in this section) patent license to make, have made, + * use, offer to sell, sell, import, and otherwise transfer the Work, + * where such license applies only to those patent claims licensable + * by such Contributor that are necessarily infringed by their + * Contribution(s) alone or by combination of their Contribution(s) + * with the Work to which such Contribution(s) was submitted. If You + * institute patent litigation against any entity (including a + * cross-claim or counterclaim in a lawsuit) alleging that the Work + * or a Contribution incorporated within the Work constitutes direct + * or contributory patent infringement, then any patent licenses + * granted to You under this License for that Work shall terminate + * as of the date such litigation is filed. + * + * 4. Redistribution. You may reproduce and distribute copies of the + * Work or Derivative Works thereof in any medium, with or without + * modifications, and in Source or Object form, provided that You + * meet the following conditions: + * + * (a) You must give any other recipients of the Work or + * Derivative Works a copy of this License; and + * + * (b) You must cause any modified files to carry prominent notices + * stating that You changed the files; and + * + * (c) You must retain, in the Source form of any Derivative Works + * that You distribute, all copyright, patent, trademark, and + * attribution notices from the Source form of the Work, + * excluding those notices that do not pertain to any part of + * the Derivative Works; and + * + * (d) If the Work includes a "NOTICE" text file as part of its + * distribution, then any Derivative Works that You distribute must + * include a readable copy of the attribution notices contained + * within such NOTICE file, excluding those notices that do not + * pertain to any part of the Derivative Works, in at least one + * of the following places: within a NOTICE text file distributed + * as part of the Derivative Works; within the Source form or + * documentation, if provided along with the Derivative Works; or, + * within a display generated by the Derivative Works, if and + * wherever such third-party notices normally appear. The contents + * of the NOTICE file are for informational purposes only and + * do not modify the License. You may add Your own attribution + * notices within Derivative Works that You distribute, alongside + * or as an addendum to the NOTICE text from the Work, provided + * that such additional attribution notices cannot be construed + * as modifying the License. + * + * You may add Your own copyright statement to Your modifications and + * may provide additional or different license terms and conditions + * for use, reproduction, or distribution of Your modifications, or + * for any such Derivative Works as a whole, provided Your use, + * reproduction, and distribution of the Work otherwise complies with + * the conditions stated in this License. + * + * 5. Submission of Contributions. Unless You explicitly state otherwise, + * any Contribution intentionally submitted for inclusion in the Work + * by You to the Licensor shall be under the terms and conditions of + * this License, without any additional terms or conditions. + * Notwithstanding the above, nothing herein shall supersede or modify + * the terms of any separate license agreement you may have executed + * with Licensor regarding such Contributions. + * + * 6. Trademarks. This License does not grant permission to use the trade + * names, trademarks, service marks, or product names of the Licensor, + * except as required for reasonable and customary use in describing the + * origin of the Work and reproducing the content of the NOTICE file. + * + * 7. Disclaimer of Warranty. Unless required by applicable law or + * agreed to in writing, Licensor provides the Work (and each + * Contributor provides its Contributions) on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied, including, without limitation, any warranties or conditions + * of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + * PARTICULAR PURPOSE. You are solely responsible for determining the + * appropriateness of using or redistributing the Work and assume any + * risks associated with Your exercise of permissions under this License. + * + * 8. Limitation of Liability. In no event and under no legal theory, + * whether in tort (including negligence), contract, or otherwise, + * unless required by applicable law (such as deliberate and grossly + * negligent acts) or agreed to in writing, shall any Contributor be + * liable to You for damages, including any direct, indirect, special, + * incidental, or consequential damages of any character arising as a + * result of this License or out of the use or inability to use the + * Work (including but not limited to damages for loss of goodwill, + * work stoppage, computer failure or malfunction, or any and all + * other commercial damages or losses), even if such Contributor + * has been advised of the possibility of such damages. + * + * 9. Accepting Warranty or Additional Liability. While redistributing + * the Work or Derivative Works thereof, You may choose to offer, + * and charge a fee for, acceptance of support, warranty, indemnity, + * or other liability obligations and/or rights consistent with this + * License. However, in accepting such obligations, You may act only + * on Your own behalf and on Your sole responsibility, not on behalf + * of any other Contributor, and only if You agree to indemnify, + * defend, and hold each Contributor harmless for any liability + * incurred by, or claims asserted against, such Contributor by reason + * of your accepting any such warranty or additional liability. + * + * END OF TERMS AND CONDITIONS + * + * APPENDIX: How to apply the Apache License to your work. + * + * To apply the Apache License to your work, attach the following + * boilerplate notice, with the fields enclosed by brackets "[]" + * replaced with your own identifying information. (Don't include + * the brackets!) The text should be enclosed in the appropriate + * comment syntax for the file format. We also recommend that a + * file or class name and description of purpose be included on the + * same "printed page" as the copyright notice for easier + * identification within third-party archives. + * + * Copyright [yyyy] [name of copyright owner] + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- From LICENSE-commons-lang.txt: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-- From LICENSE-docbook.txt: + + + +-- From LICENSE-dom4j.txt: + +BSD style license + +Redistribution and use of this software and associated documentation +("Software"), with or without modification, are permitted provided that +the following conditions are met: + +1. Redistributions of source code must retain copyright statements +and notices. Redistributions must also contain a copy of this +document. + +2. Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following +disclaimer in the documentation and/or other materials provided +with the distribution. + +3. The name "DOM4J" must not be used to endorse or promote +products derived from this Software without prior written +permission of MetaStuff, Ltd. For written permission, please +contact dom4j-info@metastuff.com. + +4. Products derived from this Software may not be called "DOM4J" +nor may "DOM4J" appear in their names without prior written +permission of MetaStuff, Ltd. DOM4J is a registered trademark of +MetaStuff, Ltd. + +5. Due credit should be given to the DOM4J Project +(http://dom4j.org/). + +THIS SOFTWARE IS PROVIDED BY METASTUFF, LTD. AND CONTRIBUTORS ``AS IS'' +AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL METASTUFF, LTD. OR ITS +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Copyright 2001 (C) MetaStuff, Ltd. All Rights Reserved. + +-- From LICENSE-jFormatString.txt: + +The GNU General Public License (GPL) + +Version 2, June 1991 + +Copyright (C) 1989, 1991 Free Software Foundation, Inc. +59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +Everyone is permitted to copy and distribute verbatim copies of this license +document, but changing it is not allowed. + +Preamble + +The licenses for most software are designed to take away your freedom to share +and change it. By contrast, the GNU General Public License is intended to +guarantee your freedom to share and change free software--to make sure the +software is free for all its users. This General Public License applies to +most of the Free Software Foundation's software and to any other program whose +authors commit to using it. (Some other Free Software Foundation software is +covered by the GNU Library General Public License instead.) You can apply it to +your programs, too. + +When we speak of free software, we are referring to freedom, not price. Our +General Public Licenses are designed to make sure that you have the freedom to +distribute copies of free software (and charge for this service if you wish), +that you receive source code or can get it if you want it, that you can change +the software or use pieces of it in new free programs; and that you know you +can do these things. + +To protect your rights, we need to make restrictions that forbid anyone to deny +you these rights or to ask you to surrender the rights. These restrictions +translate to certain responsibilities for you if you distribute copies of the +software, or if you modify it. + +For example, if you distribute copies of such a program, whether gratis or for +a fee, you must give the recipients all the rights that you have. You must +make sure that they, too, receive or can get the source code. And you must +show them these terms so they know their rights. + +We protect your rights with two steps: (1) copyright the software, and (2) +offer you this license which gives you legal permission to copy, distribute +and/or modify the software. + +Also, for each author's protection and ours, we want to make certain that +everyone understands that there is no warranty for this free software. If the +software is modified by someone else and passed on, we want its recipients to +know that what they have is not the original, so that any problems introduced +by others will not reflect on the original authors' reputations. + +Finally, any free program is threatened constantly by software patents. We +wish to avoid the danger that redistributors of a free program will +individually obtain patent licenses, in effect making the program proprietary. +To prevent this, we have made it clear that any patent must be licensed for +everyone's free use or not licensed at all. + +The precise terms and conditions for copying, distribution and modification +follow. + +TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + +0. This License applies to any program or other work which contains a notice +placed by the copyright holder saying it may be distributed under the terms of +this General Public License. The "Program", below, refers to any such program +or work, and a "work based on the Program" means either the Program or any +derivative work under copyright law: that is to say, a work containing the +Program or a portion of it, either verbatim or with modifications and/or +translated into another language. (Hereinafter, translation is included +without limitation in the term "modification".) Each licensee is addressed as +"you". + +Activities other than copying, distribution and modification are not covered by +this License; they are outside its scope. The act of running the Program is +not restricted, and the output from the Program is covered only if its contents +constitute a work based on the Program (independent of having been made by +running the Program). Whether that is true depends on what the Program does. + +1. You may copy and distribute verbatim copies of the Program's source code as +you receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice and +disclaimer of warranty; keep intact all the notices that refer to this License +and to the absence of any warranty; and give any other recipients of the +Program a copy of this License along with the Program. + +You may charge a fee for the physical act of transferring a copy, and you may +at your option offer warranty protection in exchange for a fee. + +2. You may modify your copy or copies of the Program or any portion of it, thus +forming a work based on the Program, and copy and distribute such modifications +or work under the terms of Section 1 above, provided that you also meet all of +these conditions: + +a) You must cause the modified files to carry prominent notices stating +that you changed the files and the date of any change. + +b) You must cause any work that you distribute or publish, that in whole or +in part contains or is derived from the Program or any part thereof, to be +licensed as a whole at no charge to all third parties under the terms of +this License. + +c) If the modified program normally reads commands interactively when run, +you must cause it, when started running for such interactive use in the +most ordinary way, to print or display an announcement including an +appropriate copyright notice and a notice that there is no warranty (or +else, saying that you provide a warranty) and that users may redistribute +the program under these conditions, and telling the user how to view a copy +of this License. (Exception: if the Program itself is interactive but does +not normally print such an announcement, your work based on the Program is +not required to print an announcement.) + +These requirements apply to the modified work as a whole. If identifiable +sections of that work are not derived from the Program, and can be reasonably +considered independent and separate works in themselves, then this License, and +its terms, do not apply to those sections when you distribute them as separate +works. But when you distribute the same sections as part of a whole which is a +work based on the Program, the distribution of the whole must be on the terms +of this License, whose permissions for other licensees extend to the entire +whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest your +rights to work written entirely by you; rather, the intent is to exercise the +right to control the distribution of derivative or collective works based on +the Program. + +In addition, mere aggregation of another work not based on the Program with the +Program (or with a work based on the Program) on a volume of a storage or +distribution medium does not bring the other work under the scope of this +License. + +3. You may copy and distribute the Program (or a work based on it, under +Section 2) in object code or executable form under the terms of Sections 1 and +2 above provided that you also do one of the following: + +a) Accompany it with the complete corresponding machine-readable source +code, which must be distributed under the terms of Sections 1 and 2 above +on a medium customarily used for software interchange; or, + +b) Accompany it with a written offer, valid for at least three years, to +give any third party, for a charge no more than your cost of physically +performing source distribution, a complete machine-readable copy of the +corresponding source code, to be distributed under the terms of Sections 1 +and 2 above on a medium customarily used for software interchange; or, + +c) Accompany it with the information you received as to the offer to +distribute corresponding source code. (This alternative is allowed only +for noncommercial distribution and only if you received the program in +object code or executable form with such an offer, in accord with +Subsection b above.) + +The source code for a work means the preferred form of the work for making +modifications to it. For an executable work, complete source code means all +the source code for all modules it contains, plus any associated interface +definition files, plus the scripts used to control compilation and installation +of the executable. However, as a special exception, the source code +distributed need not include anything that is normally distributed (in either +source or binary form) with the major components (compiler, kernel, and so on) +of the operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the source +code from the same place counts as distribution of the source code, even though +third parties are not compelled to copy the source along with the object code. + +4. You may not copy, modify, sublicense, or distribute the Program except as +expressly provided under this License. Any attempt otherwise to copy, modify, +sublicense or distribute the Program is void, and will automatically terminate +your rights under this License. However, parties who have received copies, or +rights, from you under this License will not have their licenses terminated so +long as such parties remain in full compliance. + +5. You are not required to accept this License, since you have not signed it. +However, nothing else grants you permission to modify or distribute the Program +or its derivative works. These actions are prohibited by law if you do not +accept this License. Therefore, by modifying or distributing the Program (or +any work based on the Program), you indicate your acceptance of this License to +do so, and all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + +6. Each time you redistribute the Program (or any work based on the Program), +the recipient automatically receives a license from the original licensor to +copy, distribute or modify the Program subject to these terms and conditions. +You may not impose any further restrictions on the recipients' exercise of the +rights granted herein. You are not responsible for enforcing compliance by +third parties to this License. + +7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), conditions +are imposed on you (whether by court order, agreement or otherwise) that +contradict the conditions of this License, they do not excuse you from the +conditions of this License. If you cannot distribute so as to satisfy +simultaneously your obligations under this License and any other pertinent +obligations, then as a consequence you may not distribute the Program at all. +For example, if a patent license would not permit royalty-free redistribution +of the Program by all those who receive copies directly or indirectly through +you, then the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply and +the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any patents or +other property right claims or to contest validity of any such claims; this +section has the sole purpose of protecting the integrity of the free software +distribution system, which is implemented by public license practices. Many +people have made generous contributions to the wide range of software +distributed through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing to +distribute software through any other system and a licensee cannot impose that +choice. + +This section is intended to make thoroughly clear what is believed to be a +consequence of the rest of this License. + +8. If the distribution and/or use of the Program is restricted in certain +countries either by patents or by copyrighted interfaces, the original +copyright holder who places the Program under this License may add an explicit +geographical distribution limitation excluding those countries, so that +distribution is permitted only in or among countries not thus excluded. In +such case, this License incorporates the limitation as if written in the body +of this License. + +9. The Free Software Foundation may publish revised and/or new versions of the +General Public License from time to time. Such new versions will be similar in +spirit to the present version, but may differ in detail to address new problems +or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any later +version", you have the option of following the terms and conditions either of +that version or of any later version published by the Free Software Foundation. +If the Program does not specify a version number of this License, you may +choose any version ever published by the Free Software Foundation. + +10. If you wish to incorporate parts of the Program into other free programs +whose distribution conditions are different, write to the author to ask for +permission. For software which is copyrighted by the Free Software Foundation, +write to the Free Software Foundation; we sometimes make exceptions for this. +Our decision will be guided by the two goals of preserving the free status of +all derivatives of our free software and of promoting the sharing and reuse of +software generally. + +NO WARRANTY + +11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR +THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE +STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE +PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND +PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, +YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL +ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE +PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR +INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA +BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER +OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +END OF TERMS AND CONDITIONS + +How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest possible +use to the public, the best way to achieve this is to make it free software +which everyone can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest to attach +them to the start of each source file to most effectively convey the exclusion +of warranty; and each file should have at least the "copyright" line and a +pointer to where the full notice is found. + +One line to give the program's name and a brief idea of what it does. + +Copyright (C) + +This program is free software; you can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2 of the License, or (at your option) +any later version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., 59 +Temple Place, Suite 330, Boston, MA 02111-1307 USA + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this when it +starts in an interactive mode: + +Gnomovision version 69, Copyright (C) year name of author Gnomovision comes +with ABSOLUTELY NO WARRANTY; for details type 'show w'. This is free +software, and you are welcome to redistribute it under certain conditions; +type 'show c' for details. + +The hypothetical commands 'show w' and 'show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may be +called something other than 'show w' and 'show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your school, +if any, to sign a "copyright disclaimer" for the program, if necessary. Here +is a sample; alter the names: + +Yoyodyne, Inc., hereby disclaims all copyright interest in the program +'Gnomovision' (which makes passes at compilers) written by James Hacker. + +signature of Ty Coon, 1 April 1989 + +Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General Public +License instead of this License. + + +"CLASSPATH" EXCEPTION TO THE GPL + +Certain source files distributed by Sun Microsystems, Inc. are subject to +the following clarification and special exception to the GPL, but only where +Sun has expressly included in the particular source file's header the words +"Sun designates this particular file as subject to the "Classpath" exception +as provided by Sun in the LICENSE file that accompanied this code." + +Linking this library statically or dynamically with other modules is making +a combined work based on this library. Thus, the terms and conditions of +the GNU General Public License cover the whole combination. + +As a special exception, the copyright holders of this library give you +permission to link this library with independent modules to produce an +executable, regardless of the license terms of these independent modules, +and to copy and distribute the resulting executable under terms of your +choice, provided that you also meet, for each linked independent module, +the terms and conditions of the license of that module. An independent +module is a module which is not derived from or based on this library. If +you modify this library, you may extend this exception to your version of +the library, but you are not obligated to do so. If you do not wish to do +so, delete this exception statement from your version. + +-- From LICENSE-jaxen.txt: + +/* +$Id: LICENSE-jaxen.txt,v 1.1 2008/06/18 18:54:23 wpugh Exp $ + +Copyright 2003-2006 The Werken Company. All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +* Neither the name of the Jaxen Project nor the names of its +contributors may be used to endorse or promote products derived +from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ + +-- From LICENSE-jcip.txt: + +The Java code in the package net.jcip.annotations +is copyright (c) 2005 Brian Goetz +and is released under the Creative Commons Attribution License +(http://creativecommons.org/licenses/by/2.5) +Official home: http://www.jcip.net + +-- From LICENSE-jdepend.txt: + +The jdepend library (lib/jdepend-2.9.jar) is distributed under the terms of the BSD license: +http://www.clarkware.com/software/JDepend.html#license +http://www.clarkware.com/software/license.txt + +Copyright (C) 2001 Clarkware Consulting, Inc. +All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +3. Neither the name of Clarkware Consulting, Inc. nor the names of its +contributors may be used to endorse or promote products derived +from this software without prior written permission. For written +permission, please contact clarkware@clarkware.com. + +THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +CLARKWARE CONSULTING OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-- From LICENSE-jsr305.txt: + +The JSR-305 reference implementation (lib/jsr305.jar) is +distributed under the terms of the New BSD license: + +http://www.opensource.org/licenses/bsd-license.php + +See the JSR-305 home page for more information: + +http://code.google.com/p/jsr-305/ + +----- + +The following software may be included in this product: floatingdecimal. The source code is available at https://github.com/airlift/floatingdecimal. You may also request a copy of the source code by sending a request to opensource@fb.com. This software contains the following license and notice below: + +/* +* Copyright (c) 1996, 2011, Oracle and/or its affiliates. All rights reserved. +* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +* +* This code is free software; you can redistribute it and/or modify it +* under the terms of the GNU General Public License version 2 only, as +* published by the Free Software Foundation. Oracle designates this +* particular file as subject to the "Classpath" exception as provided +* by Oracle in the LICENSE file that accompanied this code. +* +* This code is distributed in the hope that it will be useful, but WITHOUT +* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +* version 2 for more details (a copy is included in the LICENSE file that +* accompanied this code). +* +* You should have received a copy of the GNU General Public License version +* 2 along with this work; if not, write to the Free Software Foundation, +* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +* +* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +* or visit www.oracle.com if you need additional information or have any +* questions. +*/ + + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. + +----- + +The following software may be included in this product: H2 Database Engine. The source code is available at http://www.h2database.com/html/download.html. You may also request a copy of the source code by sending a request to opensource@fb.com. This software contains the following license and notice below: + +Eclipse Public License - v 1.0 + +THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + +a) in the case of the initial Contributor, the initial code and documentation distributed under this Agreement, and + +b) in the case of each subsequent Contributor: + +i) changes to the Program, and + +ii) additions to the Program; + +where such changes and/or additions to the Program originate from and are distributed by that particular Contributor. A Contribution 'originates' from a Contributor if it was added to the Program by such Contributor itself or anyone acting on such Contributor's behalf. Contributions do not include additions to the Program which: (i) are separate modules of software distributed in conjunction with the Program under their own license agreement, and (ii) are not derivative works of the Program. + +"Contributor" means any person or entity that distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor which are necessarily infringed by the use or sale of its Contribution alone or when combined with the Program. + +"Program" means the Contributions distributed in accordance with this Agreement. + +"Recipient" means anyone who receives the Program under this Agreement, including all Contributors. + +2. GRANT OF RIGHTS + +a) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, distribute and sublicense the Contribution of such Contributor, if any, and such derivative works, in source code and object code form. + +b) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed Patents to make, use, sell, offer to sell, import and otherwise transfer the Contribution of such Contributor, if any, in source code and object code form. This patent license shall apply to the combination of the Contribution and the Program if, at the time the Contribution is added by the Contributor, such addition of the Contribution causes such combination to be covered by the Licensed Patents. The patent license shall not apply to any other combinations which include the Contribution. No hardware per se is licensed hereunder. + +c) Recipient understands that although each Contributor grants the licenses to its Contributions set forth herein, no assurances are provided by any Contributor that the Program does not infringe the patent or other intellectual property rights of any other entity. Each Contributor disclaims any liability to Recipient for claims brought by any other entity based on infringement of intellectual property rights or otherwise. As a condition to exercising the rights and licenses granted hereunder, each Recipient hereby assumes sole responsibility to secure any other intellectual property rights needed, if any. For example, if a third party patent license is required to allow Recipient to distribute the Program, it is Recipient's responsibility to acquire that license before distributing the Program. + +d) Each Contributor represents that to its knowledge it has sufficient copyright rights in its Contribution, if any, to grant the copyright license set forth in this Agreement. + +3. REQUIREMENTS + +A Contributor may choose to distribute the Program in object code form under its own license agreement, provided that: + +a) it complies with the terms and conditions of this Agreement; and + +b) its license agreement: + +i) effectively disclaims on behalf of all Contributors all warranties and conditions, express and implied, including warranties or conditions of title and non-infringement, and implied warranties or conditions of merchantability and fitness for a particular purpose; + +ii) effectively excludes on behalf of all Contributors all liability for damages, including direct, indirect, special, incidental and consequential damages, such as lost profits; + +iii) states that any provisions which differ from this Agreement are offered by that Contributor alone and not by any other party; and + +iv) states that source code for the Program is available from such Contributor, and informs licensees how to obtain it in a reasonable manner on or through a medium customarily used for software exchange. + +When the Program is made available in source code form: + +a) it must be made available under this Agreement; and + +b) a copy of this Agreement must be included with each copy of the Program. + +Contributors may not remove or alter any copyright notices contained within the Program. + +Each Contributor must identify itself as the originator of its Contribution, if any, in a manner that reasonably allows subsequent Recipients to identify the originator of the Contribution. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities with respect to end users, business partners and the like. While this license is intended to facilitate the commercial use of the Program, the Contributor who includes the Program in a commercial product offering should do so in a manner which does not create potential liability for other Contributors. Therefore, if a Contributor includes the Program in a commercial product offering, such Contributor ("Commercial Contributor") hereby agrees to defend and indemnify every other Contributor ("Indemnified Contributor") against any losses, damages and costs (collectively "Losses") arising from claims, lawsuits and other legal actions brought by a third party against the Indemnified Contributor to the extent caused by the acts or omissions of such Commercial Contributor in connection with its distribution of the Program in a commercial product offering. The obligations in this section do not apply to any claims or Losses relating to any actual or alleged intellectual property infringement. In order to qualify, an Indemnified Contributor must: a) promptly notify the Commercial Contributor in writing of such claim, and b) allow the Commercial Contributor to control, and cooperate with the Commercial Contributor in, the defense and any related settlement negotiations. The Indemnified Contributor may participate in any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial product offering, Product X. That Contributor is then a Commercial Contributor. If that Commercial Contributor then makes performance claims, or offers warranties related to Product X, those performance claims and warranties are such Commercial Contributor's responsibility alone. Under this section, the Commercial Contributor would have to defend claims against the other Contributors related to those performance claims and warranties, and if a court requires any other Contributor to pay any damages as a result, the Commercial Contributor must pay those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the appropriateness of using and distributing the Program and assumes all risks associated with its exercise of rights under this Agreement , including but not limited to the risks and costs of program errors, compliance with applicable laws, damage to or loss of data, programs or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this Agreement, and without further action by the parties hereto, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable. + +If Recipient institutes patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Program itself (excluding combinations of the Program with other software or hardware) infringes such Recipient's patent(s), then such Recipient's rights granted under Section 2(b) shall terminate as of the date such litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it fails to comply with any of the material terms or conditions of this Agreement and does not cure such failure in a reasonable period of time after becoming aware of such noncompliance. If all Recipient's rights under this Agreement terminate, Recipient agrees to cease use and distribution of the Program as soon as reasonably practicable. However, Recipient's obligations under this Agreement and any licenses granted by Recipient relating to the Program shall continue and survive. + +Everyone is permitted to copy and distribute copies of this Agreement, but in order to avoid inconsistency the Agreement is copyrighted and may only be modified in the following manner. The Agreement Steward reserves the right to publish new versions (including revisions) of this Agreement from time to time. No one other than the Agreement Steward has the right to modify this Agreement. The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation may assign the responsibility to serve as the Agreement Steward to a suitable separate entity. Each new version of the Agreement will be given a distinguishing version number. The Program (including Contributions) may always be distributed subject to the version of the Agreement under which it was received. In addition, after a new version of the Agreement is published, Contributor may elect to distribute the Program (including its Contributions) under the new version. Except as expressly stated in Sections 2(a) and 2(b) above, Recipient receives no rights or licenses to the intellectual property of any Contributor under this Agreement, whether expressly, by implication, estoppel or otherwise. All rights in the Program not expressly granted under this Agreement are reserved. + +This Agreement is governed by the laws of the State of New York and the intellectual property laws of the United States of America. No party to this Agreement will bring a legal action under this Agreement more than one year after the cause of action arose. Each party waives its rights to a jury trial in any resulting litigation. + +----- + +The following software may be included in this product: java servlet api. The source code is available at https://java.net/projects/servlet-spec/. You may also request a copy of the source code by sending a request to opensource@fb.com. This software contains the following license and notice below: + +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL - Version 1.1) +1. Definitions. + + 1.1. “Contributor” means each individual or entity that creates or contributes to the creation of Modifications. + + 1.2. “Contributor Version” means the combination of the Original Software, prior Modifications used by a Contributor (if any), and the Modifications made by that particular Contributor. + + 1.3. “Covered Software” means (a) the Original Software, or (b) Modifications, or (c) the combination of files containing Original Software with files containing Modifications, in each case including portions thereof. + + 1.4. “Executable” means the Covered Software in any form other than Source Code. + + 1.5. “Initial Developer” means the individual or entity that first makes Original Software available under this License. + + 1.6. “Larger Work” means a work which combines Covered Software or portions thereof with code not governed by the terms of this License. + + 1.7. “License” means this document. + + 1.8. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently acquired, any and all of the rights conveyed herein. + + 1.9. “Modifications” means the Source Code and Executable form of any of the following: + + A. Any file that results from an addition to, deletion from or modification of the contents of a file containing Original Software or previous Modifications; + + B. Any new file that contains any part of the Original Software or previous Modification; or + + C. Any new file that is contributed or otherwise made available under the terms of this License. + + 1.10. “Original Software” means the Source Code and Executable form of computer software code that is originally released under this License. + + 1.11. “Patent Claims” means any patent claim(s), now owned or hereafter acquired, including without limitation, method, process, and apparatus claims, in any patent Licensable by grantor. + + 1.12. “Source Code” means (a) the common form of computer software code in which modifications are made and (b) associated documentation included in or with such code. + + 1.13. “You” (or “Your”) means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, “You” includes any entity which controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. + +2. License Grants. + + 2.1. The Initial Developer Grant. + + Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, the Initial Developer hereby grants You a world-wide, royalty-free, non-exclusive license: + + (a) under intellectual property rights (other than patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, display, perform, sublicense and distribute the Original Software (or portions thereof), with or without Modifications, and/or as part of a Larger Work; and + + (b) under Patent Claims infringed by the making, using or selling of Original Software, to make, have made, use, practice, sell, and offer for sale, and/or otherwise dispose of the Original Software (or portions thereof). + + (c) The licenses granted in Sections 2.1(a) and (b) are effective on the date Initial Developer first distributes or otherwise makes the Original Software available to a third party under the terms of this License. + + (d) Notwithstanding Section 2.1(b) above, no patent license is granted: (1) for code that You delete from the Original Software, or (2) for infringements caused by: (i) the modification of the Original Software, or (ii) the combination of the Original Software with other software or devices. + + 2.2. Contributor Grant. + + Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: + + (a) under intellectual property rights (other than patent or trademark) Licensable by Contributor to use, reproduce, modify, display, perform, sublicense and distribute the Modifications created by such Contributor (or portions thereof), either on an unmodified basis, with other Modifications, as Covered Software and/or as part of a Larger Work; and + + (b) under Patent Claims infringed by the making, using, or selling of Modifications made by that Contributor either alone and/or in combination with its Contributor Version (or portions of such combination), to make, use, sell, offer for sale, have made, and/or otherwise dispose of: (1) Modifications made by that Contributor (or portions thereof); and (2) the combination of Modifications made by that Contributor with its Contributor Version (or portions of such combination). + + (c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first distributes or otherwise makes the Modifications available to a third party. + + (d) Notwithstanding Section 2.2(b) above, no patent license is granted: (1) for any code that Contributor has deleted from the Contributor Version; (2) for infringements caused by: (i) third party modifications of Contributor Version, or (ii) the combination of Modifications made by that Contributor with other software (except as part of the Contributor Version) or other devices; or (3) under Patent Claims infringed by Covered Software in the absence of Modifications made by that Contributor. + +3. Distribution Obligations. + + 3.1. Availability of Source Code. + + Any Covered Software that You distribute or otherwise make available in Executable form must also be made available in Source Code form and that Source Code form must be distributed only under the terms of this License. You must include a copy of this License with every copy of the Source Code form of the Covered Software You distribute or otherwise make available. You must inform recipients of any such Covered Software in Executable form as to how they can obtain such Covered Software in Source Code form in a reasonable manner on or through a medium customarily used for software exchange. + + 3.2. Modifications. + + The Modifications that You create or to which You contribute are governed by the terms of this License. You represent that You believe Your Modifications are Your original creation(s) and/or You have sufficient rights to grant the rights conveyed by this License. + + 3.3. Required Notices. + + You must include a notice in each of Your Modifications that identifies You as the Contributor of the Modification. You may not remove or alter any copyright, patent or trademark notices contained within the Covered Software, or any notices of licensing or any descriptive text giving attribution to any Contributor or the Initial Developer. + + 3.4. Application of Additional Terms. + + You may not offer or impose any terms on any Covered Software in Source Code form that alters or restricts the applicable version of this License or the recipients’ rights hereunder. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, you may do so only on Your own behalf, and not on behalf of the Initial Developer or any Contributor. You must make it absolutely clear that any such warranty, support, indemnity or liability obligation is offered by You alone, and You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of warranty, support, indemnity or liability terms You offer. + + 3.5. Distribution of Executable Versions. + + You may distribute the Executable form of the Covered Software under the terms of this License or under the terms of a license of Your choice, which may contain terms different from this License, provided that You are in compliance with the terms of this License and that the license for the Executable form does not attempt to limit or alter the recipient’s rights in the Source Code form from the rights set forth in this License. If You distribute the Covered Software in Executable form under a different license, You must make it absolutely clear that any terms which differ from this License are offered by You alone, not by the Initial Developer or Contributor. You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of any such terms You offer. + + 3.6. Larger Works. + + You may create a Larger Work by combining Covered Software with other code not governed by the terms of this License and distribute the Larger Work as a single product. In such a case, You must make sure the requirements of this License are fulfilled for the Covered Software. + +4. Versions of the License. + + 4.1. New Versions. + + Oracle is the initial license steward and may publish revised and/or new versions of this License from time to time. Each version will be given a distinguishing version number. Except as provided in Section 4.3, no one other than the license steward has the right to modify this License. + + 4.2. Effect of New Versions. + + You may always continue to use, distribute or otherwise make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. If the Initial Developer includes a notice in the Original Software prohibiting it from being distributed or otherwise made available under any subsequent version of the License, You must distribute and make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. Otherwise, You may also choose to use, distribute or otherwise make the Covered Software available under the terms of any subsequent version of the License published by the license steward. + + 4.3. Modified Versions. + + When You are an Initial Developer and You want to create a new license for Your Original Software, You may create and use a modified version of this License if You: (a) rename the license and remove any references to the name of the license steward (except to note that the license differs from this License); and (b) otherwise make it clear that the license contains terms which differ from this License. + +5. DISCLAIMER OF WARRANTY. + +COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. +6. TERMINATION. + + 6.1. This License and the rights granted hereunder will terminate automatically if You fail to comply with terms herein and fail to cure such breach within 30 days of becoming aware of the breach. Provisions which, by their nature, must remain in effect beyond the termination of this License shall survive. + + 6.2. If You assert a patent infringement claim (excluding declaratory judgment actions) against Initial Developer or a Contributor (the Initial Developer or Contributor against whom You assert such claim is referred to as “Participant”) alleging that the Participant Software (meaning the Contributor Version where the Participant is a Contributor or the Original Software where the Participant is the Initial Developer) directly or indirectly infringes any patent, then any and all rights granted directly or indirectly to You by such Participant, the Initial Developer (if the Initial Developer is not the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from Participant terminate prospectively and automatically at the expiration of such 60 day notice period, unless if within such 60 day period You withdraw Your claim with respect to the Participant Software against such Participant either unilaterally or pursuant to a written agreement with Participant. + + 6.3. If You assert a patent infringement claim against Participant alleging that the Participant Software directly or indirectly infringes any patent where such claim is resolved (such as by license or settlement) prior to the initiation of patent infringement litigation, then the reasonable value of the licenses granted by such Participant under Sections 2.1 or 2.2 shall be taken into account in determining the amount or value of any payment or license. + + 6.4. In the event of termination under Sections 6.1 or 6.2 above, all end user licenses that have been validly granted by You or any distributor hereunder prior to termination (excluding licenses granted to You by any distributor) shall survive termination. + +7. LIMITATION OF LIABILITY. + +UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTY’S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. +8. U.S. GOVERNMENT END USERS. + +The Covered Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of “commercial computer software” (as that term is defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer software documentation” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software with only those rights set forth herein. This U.S. Government Rights clause is in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision that addresses Government rights in computer software under this License. +9. MISCELLANEOUS. + +This License represents the complete agreement concerning subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. This License shall be governed by the law of the jurisdiction specified in a notice contained within the Original Software (except to the extent applicable law, if any, provides otherwise), excluding such jurisdiction’s conflict-of-law provisions. Any litigation relating to this License shall be subject to the jurisdiction of the courts located in the jurisdiction and venue specified in a notice contained within the Original Software, with the losing party responsible for costs, including, without limitation, court costs and reasonable attorneys’ fees and expenses. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not apply to this License. You agree that You alone are responsible for compliance with the United States export administration regulations (and the export control laws and regulation of any other countries) when You use, distribute or otherwise make available any Covered Software. +10. RESPONSIBILITY FOR CLAIMS. + +As between Initial Developer and the Contributors, each party is responsible for claims and damages arising, directly or indirectly, out of its utilization of rights under this License and You agree to work with Initial Developer and Contributors to distribute such responsibility on an equitable basis. Nothing herein is intended or shall be deemed to constitute any admission of liability. + +----- + +The following software may be included in this product: JCodings. This software contains the following license and notice below: + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +----- + +The following software may be included in this product: jersey. The source code is available at https://jersey.java.net/download.html. You may also request a copy of the source code by sending a request to opensource@fb.com. This software contains the following license and notice below: + +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL - Version 1.1) +1. Definitions. + + 1.1. “Contributor” means each individual or entity that creates or contributes to the creation of Modifications. + + 1.2. “Contributor Version” means the combination of the Original Software, prior Modifications used by a Contributor (if any), and the Modifications made by that particular Contributor. + + 1.3. “Covered Software” means (a) the Original Software, or (b) Modifications, or (c) the combination of files containing Original Software with files containing Modifications, in each case including portions thereof. + + 1.4. “Executable” means the Covered Software in any form other than Source Code. + + 1.5. “Initial Developer” means the individual or entity that first makes Original Software available under this License. + + 1.6. “Larger Work” means a work which combines Covered Software or portions thereof with code not governed by the terms of this License. + + 1.7. “License” means this document. + + 1.8. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently acquired, any and all of the rights conveyed herein. + + 1.9. “Modifications” means the Source Code and Executable form of any of the following: + + A. Any file that results from an addition to, deletion from or modification of the contents of a file containing Original Software or previous Modifications; + + B. Any new file that contains any part of the Original Software or previous Modification; or + + C. Any new file that is contributed or otherwise made available under the terms of this License. + + 1.10. “Original Software” means the Source Code and Executable form of computer software code that is originally released under this License. + + 1.11. “Patent Claims” means any patent claim(s), now owned or hereafter acquired, including without limitation, method, process, and apparatus claims, in any patent Licensable by grantor. + + 1.12. “Source Code” means (a) the common form of computer software code in which modifications are made and (b) associated documentation included in or with such code. + + 1.13. “You” (or “Your”) means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, “You” includes any entity which controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. + +2. License Grants. + + 2.1. The Initial Developer Grant. + + Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, the Initial Developer hereby grants You a world-wide, royalty-free, non-exclusive license: + + (a) under intellectual property rights (other than patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, display, perform, sublicense and distribute the Original Software (or portions thereof), with or without Modifications, and/or as part of a Larger Work; and + + (b) under Patent Claims infringed by the making, using or selling of Original Software, to make, have made, use, practice, sell, and offer for sale, and/or otherwise dispose of the Original Software (or portions thereof). + + (c) The licenses granted in Sections 2.1(a) and (b) are effective on the date Initial Developer first distributes or otherwise makes the Original Software available to a third party under the terms of this License. + + (d) Notwithstanding Section 2.1(b) above, no patent license is granted: (1) for code that You delete from the Original Software, or (2) for infringements caused by: (i) the modification of the Original Software, or (ii) the combination of the Original Software with other software or devices. + + 2.2. Contributor Grant. + + Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: + + (a) under intellectual property rights (other than patent or trademark) Licensable by Contributor to use, reproduce, modify, display, perform, sublicense and distribute the Modifications created by such Contributor (or portions thereof), either on an unmodified basis, with other Modifications, as Covered Software and/or as part of a Larger Work; and + + (b) under Patent Claims infringed by the making, using, or selling of Modifications made by that Contributor either alone and/or in combination with its Contributor Version (or portions of such combination), to make, use, sell, offer for sale, have made, and/or otherwise dispose of: (1) Modifications made by that Contributor (or portions thereof); and (2) the combination of Modifications made by that Contributor with its Contributor Version (or portions of such combination). + + (c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first distributes or otherwise makes the Modifications available to a third party. + + (d) Notwithstanding Section 2.2(b) above, no patent license is granted: (1) for any code that Contributor has deleted from the Contributor Version; (2) for infringements caused by: (i) third party modifications of Contributor Version, or (ii) the combination of Modifications made by that Contributor with other software (except as part of the Contributor Version) or other devices; or (3) under Patent Claims infringed by Covered Software in the absence of Modifications made by that Contributor. + +3. Distribution Obligations. + + 3.1. Availability of Source Code. + + Any Covered Software that You distribute or otherwise make available in Executable form must also be made available in Source Code form and that Source Code form must be distributed only under the terms of this License. You must include a copy of this License with every copy of the Source Code form of the Covered Software You distribute or otherwise make available. You must inform recipients of any such Covered Software in Executable form as to how they can obtain such Covered Software in Source Code form in a reasonable manner on or through a medium customarily used for software exchange. + + 3.2. Modifications. + + The Modifications that You create or to which You contribute are governed by the terms of this License. You represent that You believe Your Modifications are Your original creation(s) and/or You have sufficient rights to grant the rights conveyed by this License. + + 3.3. Required Notices. + + You must include a notice in each of Your Modifications that identifies You as the Contributor of the Modification. You may not remove or alter any copyright, patent or trademark notices contained within the Covered Software, or any notices of licensing or any descriptive text giving attribution to any Contributor or the Initial Developer. + + 3.4. Application of Additional Terms. + + You may not offer or impose any terms on any Covered Software in Source Code form that alters or restricts the applicable version of this License or the recipients’ rights hereunder. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, you may do so only on Your own behalf, and not on behalf of the Initial Developer or any Contributor. You must make it absolutely clear that any such warranty, support, indemnity or liability obligation is offered by You alone, and You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of warranty, support, indemnity or liability terms You offer. + + 3.5. Distribution of Executable Versions. + + You may distribute the Executable form of the Covered Software under the terms of this License or under the terms of a license of Your choice, which may contain terms different from this License, provided that You are in compliance with the terms of this License and that the license for the Executable form does not attempt to limit or alter the recipient’s rights in the Source Code form from the rights set forth in this License. If You distribute the Covered Software in Executable form under a different license, You must make it absolutely clear that any terms which differ from this License are offered by You alone, not by the Initial Developer or Contributor. You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of any such terms You offer. + + 3.6. Larger Works. + + You may create a Larger Work by combining Covered Software with other code not governed by the terms of this License and distribute the Larger Work as a single product. In such a case, You must make sure the requirements of this License are fulfilled for the Covered Software. + +4. Versions of the License. + + 4.1. New Versions. + + Oracle is the initial license steward and may publish revised and/or new versions of this License from time to time. Each version will be given a distinguishing version number. Except as provided in Section 4.3, no one other than the license steward has the right to modify this License. + + 4.2. Effect of New Versions. + + You may always continue to use, distribute or otherwise make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. If the Initial Developer includes a notice in the Original Software prohibiting it from being distributed or otherwise made available under any subsequent version of the License, You must distribute and make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. Otherwise, You may also choose to use, distribute or otherwise make the Covered Software available under the terms of any subsequent version of the License published by the license steward. + + 4.3. Modified Versions. + + When You are an Initial Developer and You want to create a new license for Your Original Software, You may create and use a modified version of this License if You: (a) rename the license and remove any references to the name of the license steward (except to note that the license differs from this License); and (b) otherwise make it clear that the license contains terms which differ from this License. + +5. DISCLAIMER OF WARRANTY. + +COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. +6. TERMINATION. + + 6.1. This License and the rights granted hereunder will terminate automatically if You fail to comply with terms herein and fail to cure such breach within 30 days of becoming aware of the breach. Provisions which, by their nature, must remain in effect beyond the termination of this License shall survive. + + 6.2. If You assert a patent infringement claim (excluding declaratory judgment actions) against Initial Developer or a Contributor (the Initial Developer or Contributor against whom You assert such claim is referred to as “Participant”) alleging that the Participant Software (meaning the Contributor Version where the Participant is a Contributor or the Original Software where the Participant is the Initial Developer) directly or indirectly infringes any patent, then any and all rights granted directly or indirectly to You by such Participant, the Initial Developer (if the Initial Developer is not the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from Participant terminate prospectively and automatically at the expiration of such 60 day notice period, unless if within such 60 day period You withdraw Your claim with respect to the Participant Software against such Participant either unilaterally or pursuant to a written agreement with Participant. + + 6.3. If You assert a patent infringement claim against Participant alleging that the Participant Software directly or indirectly infringes any patent where such claim is resolved (such as by license or settlement) prior to the initiation of patent infringement litigation, then the reasonable value of the licenses granted by such Participant under Sections 2.1 or 2.2 shall be taken into account in determining the amount or value of any payment or license. + + 6.4. In the event of termination under Sections 6.1 or 6.2 above, all end user licenses that have been validly granted by You or any distributor hereunder prior to termination (excluding licenses granted to You by any distributor) shall survive termination. + +7. LIMITATION OF LIABILITY. + +UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTY’S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. +8. U.S. GOVERNMENT END USERS. + +The Covered Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of “commercial computer software” (as that term is defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer software documentation” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software with only those rights set forth herein. This U.S. Government Rights clause is in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision that addresses Government rights in computer software under this License. +9. MISCELLANEOUS. + +This License represents the complete agreement concerning subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. This License shall be governed by the law of the jurisdiction specified in a notice contained within the Original Software (except to the extent applicable law, if any, provides otherwise), excluding such jurisdiction’s conflict-of-law provisions. Any litigation relating to this License shall be subject to the jurisdiction of the courts located in the jurisdiction and venue specified in a notice contained within the Original Software, with the losing party responsible for costs, including, without limitation, court costs and reasonable attorneys’ fees and expenses. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not apply to this License. You agree that You alone are responsible for compliance with the United States export administration regulations (and the export control laws and regulation of any other countries) when You use, distribute or otherwise make available any Covered Software. +10. RESPONSIBILITY FOR CLAIMS. + +As between Initial Developer and the Contributors, each party is responsible for claims and damages arising, directly or indirectly, out of its utilization of rights under this License and You agree to work with Initial Developer and Contributors to distribute such responsibility on an equitable basis. Nothing herein is intended or shall be deemed to constitute any admission of liability. +NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) + +The code released under the CDDL shall be governed by the laws of the State of California (excluding conflict-of-law provisions). Any litigation relating to this License shall be subject to the jurisdiction of the Federal Courts of the Northern District of California and the state courts of the State of California, with venue lying in Santa Clara County, California. + +----- + +The following software may be included in this product: JLine. This software contains the following license and notice below: + +/* + * Copyright (c) 2002-2007, Marc Prud'hommeaux. All rights reserved. + * + * This software is distributable under the BSD license. See the terms of the + * BSD license in the documentation provided with this software. + */ + +----- + +The following software may be included in this product: Joni. This software contains the following license and notice below: + +/* +* Permission is hereby granted, free of charge, to any person obtaining a copy of +* this software and associated documentation files (the "Software"), to deal in +* the Software without restriction, including without limitation the rights to +* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +* of the Software, and to permit persons to whom the Software is furnished to do +* so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +* SOFTWARE. +*/ + +----- + +The following software may be included in this product: leveldb. This software contains the following license and notice below: + +Copyright (c) 2011 The LevelDB Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. +* Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +----- + +The following software may be included in this product: logback. The source code is available at http://logback.qos.ch/download.html. You may also request a copy of the source code by sending a request to opensource@fb.com. This software contains the following license and notice below: + +Eclipse Public License - v 1.0 + +THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + +a) in the case of the initial Contributor, the initial code and documentation distributed under this Agreement, and + +b) in the case of each subsequent Contributor: + +i) changes to the Program, and + +ii) additions to the Program; + +where such changes and/or additions to the Program originate from and are distributed by that particular Contributor. A Contribution 'originates' from a Contributor if it was added to the Program by such Contributor itself or anyone acting on such Contributor's behalf. Contributions do not include additions to the Program which: (i) are separate modules of software distributed in conjunction with the Program under their own license agreement, and (ii) are not derivative works of the Program. + +"Contributor" means any person or entity that distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor which are necessarily infringed by the use or sale of its Contribution alone or when combined with the Program. + +"Program" means the Contributions distributed in accordance with this Agreement. + +"Recipient" means anyone who receives the Program under this Agreement, including all Contributors. + +2. GRANT OF RIGHTS + +a) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, distribute and sublicense the Contribution of such Contributor, if any, and such derivative works, in source code and object code form. + +b) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed Patents to make, use, sell, offer to sell, import and otherwise transfer the Contribution of such Contributor, if any, in source code and object code form. This patent license shall apply to the combination of the Contribution and the Program if, at the time the Contribution is added by the Contributor, such addition of the Contribution causes such combination to be covered by the Licensed Patents. The patent license shall not apply to any other combinations which include the Contribution. No hardware per se is licensed hereunder. + +c) Recipient understands that although each Contributor grants the licenses to its Contributions set forth herein, no assurances are provided by any Contributor that the Program does not infringe the patent or other intellectual property rights of any other entity. Each Contributor disclaims any liability to Recipient for claims brought by any other entity based on infringement of intellectual property rights or otherwise. As a condition to exercising the rights and licenses granted hereunder, each Recipient hereby assumes sole responsibility to secure any other intellectual property rights needed, if any. For example, if a third party patent license is required to allow Recipient to distribute the Program, it is Recipient's responsibility to acquire that license before distributing the Program. + +d) Each Contributor represents that to its knowledge it has sufficient copyright rights in its Contribution, if any, to grant the copyright license set forth in this Agreement. + +3. REQUIREMENTS + +A Contributor may choose to distribute the Program in object code form under its own license agreement, provided that: + +a) it complies with the terms and conditions of this Agreement; and + +b) its license agreement: + +i) effectively disclaims on behalf of all Contributors all warranties and conditions, express and implied, including warranties or conditions of title and non-infringement, and implied warranties or conditions of merchantability and fitness for a particular purpose; + +ii) effectively excludes on behalf of all Contributors all liability for damages, including direct, indirect, special, incidental and consequential damages, such as lost profits; + +iii) states that any provisions which differ from this Agreement are offered by that Contributor alone and not by any other party; and + +iv) states that source code for the Program is available from such Contributor, and informs licensees how to obtain it in a reasonable manner on or through a medium customarily used for software exchange. + +When the Program is made available in source code form: + +a) it must be made available under this Agreement; and + +b) a copy of this Agreement must be included with each copy of the Program. + +Contributors may not remove or alter any copyright notices contained within the Program. + +Each Contributor must identify itself as the originator of its Contribution, if any, in a manner that reasonably allows subsequent Recipients to identify the originator of the Contribution. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities with respect to end users, business partners and the like. While this license is intended to facilitate the commercial use of the Program, the Contributor who includes the Program in a commercial product offering should do so in a manner which does not create potential liability for other Contributors. Therefore, if a Contributor includes the Program in a commercial product offering, such Contributor ("Commercial Contributor") hereby agrees to defend and indemnify every other Contributor ("Indemnified Contributor") against any losses, damages and costs (collectively "Losses") arising from claims, lawsuits and other legal actions brought by a third party against the Indemnified Contributor to the extent caused by the acts or omissions of such Commercial Contributor in connection with its distribution of the Program in a commercial product offering. The obligations in this section do not apply to any claims or Losses relating to any actual or alleged intellectual property infringement. In order to qualify, an Indemnified Contributor must: a) promptly notify the Commercial Contributor in writing of such claim, and b) allow the Commercial Contributor to control, and cooperate with the Commercial Contributor in, the defense and any related settlement negotiations. The Indemnified Contributor may participate in any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial product offering, Product X. That Contributor is then a Commercial Contributor. If that Commercial Contributor then makes performance claims, or offers warranties related to Product X, those performance claims and warranties are such Commercial Contributor's responsibility alone. Under this section, the Commercial Contributor would have to defend claims against the other Contributors related to those performance claims and warranties, and if a court requires any other Contributor to pay any damages as a result, the Commercial Contributor must pay those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the appropriateness of using and distributing the Program and assumes all risks associated with its exercise of rights under this Agreement , including but not limited to the risks and costs of program errors, compliance with applicable laws, damage to or loss of data, programs or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this Agreement, and without further action by the parties hereto, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable. + +If Recipient institutes patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Program itself (excluding combinations of the Program with other software or hardware) infringes such Recipient's patent(s), then such Recipient's rights granted under Section 2(b) shall terminate as of the date such litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it fails to comply with any of the material terms or conditions of this Agreement and does not cure such failure in a reasonable period of time after becoming aware of such noncompliance. If all Recipient's rights under this Agreement terminate, Recipient agrees to cease use and distribution of the Program as soon as reasonably practicable. However, Recipient's obligations under this Agreement and any licenses granted by Recipient relating to the Program shall continue and survive. + +Everyone is permitted to copy and distribute copies of this Agreement, but in order to avoid inconsistency the Agreement is copyrighted and may only be modified in the following manner. The Agreement Steward reserves the right to publish new versions (including revisions) of this Agreement from time to time. No one other than the Agreement Steward has the right to modify this Agreement. The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation may assign the responsibility to serve as the Agreement Steward to a suitable separate entity. Each new version of the Agreement will be given a distinguishing version number. The Program (including Contributions) may always be distributed subject to the version of the Agreement under which it was received. In addition, after a new version of the Agreement is published, Contributor may elect to distribute the Program (including its Contributions) under the new version. Except as expressly stated in Sections 2(a) and 2(b) above, Recipient receives no rights or licenses to the intellectual property of any Contributor under this Agreement, whether expressly, by implication, estoppel or otherwise. All rights in the Program not expressly granted under this Agreement are reserved. + +This Agreement is governed by the laws of the State of New York and the intellectual property laws of the United States of America. No party to this Agreement will bring a legal action under this Agreement more than one year after the cause of action arose. Each party waives its rights to a jury trial in any resulting litigation. + +----- + +The following software may be included in this product: protobuf. This software contains the following license and notice below: + +Copyright 2008, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. +* Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + +----- + +The following software may be included in this product: slf4j. This software contains the following license and notice below: + +Copyright (c) 2004-2013 QOS.ch +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/presto-twitter-server/README.txt b/presto-twitter-server/README.txt new file mode 100644 index 0000000000000..46017e937c7bb --- /dev/null +++ b/presto-twitter-server/README.txt @@ -0,0 +1,5 @@ +Presto is a distributed SQL query engine. + +Please see the website for installation instructions: + +https://prestodb.io/ diff --git a/presto-twitter-server/pom.xml b/presto-twitter-server/pom.xml new file mode 100644 index 0000000000000..c9b82b8eb4320 --- /dev/null +++ b/presto-twitter-server/pom.xml @@ -0,0 +1,26 @@ + + + 4.0.0 + + + com.facebook.presto + presto-root + 0.208-tw-0.53 + + + presto-twitter-server + presto-twitter-server + provisio + + + ${project.parent.basedir} + true + true + true + true + + + com.facebook.presto.server.PrestoServer + ${project.artifactId} + + diff --git a/presto-twitter-server/src/main/provisio/presto.xml b/presto-twitter-server/src/main/provisio/presto.xml new file mode 100644 index 0000000000000..7a448a5440486 --- /dev/null +++ b/presto-twitter-server/src/main/provisio/presto.xml @@ -0,0 +1,76 @@ + + + + + + + + NOTICE + README.txt + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/presto-twitter-server/src/main/provisio/twitter.xml b/presto-twitter-server/src/main/provisio/twitter.xml new file mode 100644 index 0000000000000..e1da085284c0e --- /dev/null +++ b/presto-twitter-server/src/main/provisio/twitter.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index fb0a44cd4992b..ae7be871d3772 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.208 + 0.208-tw-0.53 presto-verifier diff --git a/twitter-eventlistener-plugin/pom.xml b/twitter-eventlistener-plugin/pom.xml new file mode 100644 index 0000000000000..bf6ca6d422c4f --- /dev/null +++ b/twitter-eventlistener-plugin/pom.xml @@ -0,0 +1,115 @@ + + + 4.0.0 + + com.facebook.presto + presto-root + 0.208-tw-0.53 + + + twitter-eventlistener-plugin + Twitter Event Listener - scribes QueryCompletedEvent + presto-plugin + + + ${project.parent.basedir} + + + + + com.facebook.presto + presto-spi + 0.208-tw-0.53 + provided + + + io.airlift + log + + + io.airlift + units + provided + + + com.google.guava + guava + + + org.glassfish + javax.json + 1.0.4 + + + + + com.twitter + presto-thrift-java + 0.0.5 + + + com.twitter + util-core_2.11 + + + com.twitter + util-core-java + + + com.twitter + util-function_2.10 + + + com.twitter + util-function-java + + + commons-logging + commons-logging + + + org.scala-lang.modules + scala-parser-combinators_2.11 + + + com.twitter + scrooge-core + + + org.scala-lang + scala-library + + + org.scala-lang + scala-reflect + + + + + com.twitter + util-logging_2.10 + 6.34.0 + + + commons-logging + commons-logging + + + + + org.apache.thrift + libthrift + + + org.scala-lang + scala-library + 2.10.6 + + + commons-logging + commons-logging + + + + + diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/QueryCompletedEventScriber.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/QueryCompletedEventScriber.java new file mode 100644 index 0000000000000..6e0d31fd33529 --- /dev/null +++ b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/QueryCompletedEventScriber.java @@ -0,0 +1,114 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twitter.presto.plugin.eventlistener; + +import com.facebook.presto.spi.eventlistener.QueryCompletedEvent; +import com.facebook.presto.spi.eventlistener.QueryContext; +import com.facebook.presto.spi.eventlistener.QueryFailureInfo; +import com.facebook.presto.spi.eventlistener.QueryMetadata; +import com.facebook.presto.spi.eventlistener.QueryStatistics; +import com.twitter.presto.thriftjava.QueryCompletionEvent; +import com.twitter.presto.thriftjava.QueryState; +import io.airlift.log.Logger; +import org.apache.thrift.TException; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Class that scribes query completion events + */ +public class QueryCompletedEventScriber +{ + private static final String DASH = "-"; + private static final Logger log = Logger.get(QueryCompletedEventScriber.class); + + private TwitterScriber scriber = new TwitterScriber("presto_query_completion"); + + public void handle(QueryCompletedEvent event) + { + try { + scriber.scribe(toThriftQueryCompletionEvent(event)); + } + catch (TException e) { + log.warn(e, + String.format("Could not serialize thrift object of Query(id=%s, user=%s, env=%s, schema=%s.%s)", + event.getMetadata().getQueryId(), + event.getContext().getUser(), + event.getContext().getEnvironment(), + event.getContext().getCatalog().orElse(DASH), + event.getContext().getSchema().orElse(DASH))); + } + } + + private static QueryCompletionEvent toThriftQueryCompletionEvent(QueryCompletedEvent event) + { + QueryMetadata eventMetadata = event.getMetadata(); + QueryContext eventContext = event.getContext(); + QueryStatistics eventStat = event.getStatistics(); + + QueryCompletionEvent thriftEvent = + new com.twitter.presto.thriftjava.QueryCompletionEvent(); + + thriftEvent.setQuery_id(eventMetadata.getQueryId()); + thriftEvent.setTransaction_id(eventMetadata.getTransactionId().orElse(DASH)); + thriftEvent.setUser(eventContext.getUser()); + thriftEvent.setPrincipal(eventContext.getPrincipal().orElse(DASH)); + thriftEvent.setSource(eventContext.getSource().orElse(DASH)); + thriftEvent.setServer_version(eventContext.getServerVersion()); + thriftEvent.setEnvironment(eventContext.getEnvironment()); + thriftEvent.setCatalog(eventContext.getCatalog().orElse(DASH)); + thriftEvent.setSchema(eventContext.getSchema().orElse(DASH)); + Map> queriedColumnsByTable = new HashMap>(); + event.getIoMetadata().getInputs().forEach(input -> queriedColumnsByTable.put(String.format("%s.%s", input.getSchema(), input.getTable()), input.getColumns())); + thriftEvent.setQueried_columns_by_table(queriedColumnsByTable); + thriftEvent.setRemote_client_address(eventContext.getRemoteClientAddress().orElse(DASH)); + thriftEvent.setUser_agent(eventContext.getUserAgent().orElse(DASH)); + thriftEvent.setQuery_state(QueryState.valueOf(eventMetadata.getQueryState())); + thriftEvent.setUri(eventMetadata.getUri().toString()); + thriftEvent.setQuery(eventMetadata.getQuery()); + thriftEvent.setCreate_time_ms(event.getCreateTime().toEpochMilli()); + thriftEvent.setExecution_start_time_ms(event.getExecutionStartTime().toEpochMilli()); + thriftEvent.setEnd_time_ms(event.getEndTime().toEpochMilli()); + thriftEvent.setQueued_time_ms(eventStat.getQueuedTime().toMillis()); + thriftEvent.setQuery_wall_time_ms(eventStat.getWallTime().toMillis()); + thriftEvent.setCumulative_memory_bytesecond(eventStat.getCumulativeMemory()); + thriftEvent.setPeak_memory_bytes(eventStat.getPeakTotalNonRevocableMemoryBytes()); + thriftEvent.setCpu_time_ms(eventStat.getCpuTime().toMillis()); + if (eventStat.getAnalysisTime().isPresent()) { + thriftEvent.setAnalysis_time_ms(eventStat.getAnalysisTime().get().toMillis()); + } + if (eventStat.getDistributedPlanningTime().isPresent()) { + thriftEvent.setDistributed_planning_time_ms(eventStat.getDistributedPlanningTime().get().toMillis()); + } + thriftEvent.setTotal_bytes(eventStat.getTotalBytes()); + thriftEvent.setQuery_stages(QueryStatsHelper.getQueryStages(eventMetadata)); + thriftEvent.setOperator_summaries(QueryStatsHelper.getOperatorSummaries(eventStat)); + thriftEvent.setTotal_rows(eventStat.getTotalRows()); + thriftEvent.setSplits(eventStat.getCompletedSplits()); + if (event.getFailureInfo().isPresent()) { + QueryFailureInfo eventFailureInfo = event.getFailureInfo().get(); + thriftEvent.setError_code_id(eventFailureInfo.getErrorCode().getCode()); + thriftEvent.setError_code_name(eventFailureInfo.getErrorCode().getName()); + thriftEvent.setFailure_type(eventFailureInfo.getFailureType().orElse(DASH)); + thriftEvent.setFailure_message(eventFailureInfo.getFailureMessage().orElse(DASH)); + thriftEvent.setFailure_task(eventFailureInfo.getFailureTask().orElse(DASH)); + thriftEvent.setFailure_host(eventFailureInfo.getFailureHost().orElse(DASH)); + thriftEvent.setFailures_json(eventFailureInfo.getFailuresJson()); + } + + return thriftEvent; + } +} diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/QueryStatsHelper.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/QueryStatsHelper.java new file mode 100644 index 0000000000000..b38c5c90e9f36 --- /dev/null +++ b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/QueryStatsHelper.java @@ -0,0 +1,217 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twitter.presto.plugin.eventlistener; + +import com.facebook.presto.spi.eventlistener.QueryMetadata; +import com.facebook.presto.spi.eventlistener.QueryStatistics; +import com.twitter.presto.thriftjava.OperatorStats; +import com.twitter.presto.thriftjava.QueryStageInfo; +import io.airlift.log.Logger; +import io.airlift.units.DataSize; +import io.airlift.units.Duration; + +import javax.json.Json; +import javax.json.JsonObject; +import javax.json.JsonReader; +import javax.json.JsonValue.ValueType; + +import java.io.StringReader; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Queue; +import java.util.stream.Collectors; + +public class QueryStatsHelper +{ + private static final Logger log = Logger.get(QueryStatsHelper.class); + + private QueryStatsHelper() + { + throw new AssertionError(); + } + + private static long getBytesOrNegativeOne(String strVal) + { + try { + return DataSize.valueOf(strVal).toBytes(); + } + catch (IllegalArgumentException e) { + log.warn(e, + String.format("Failed to parse io.airlift.units.DataSize '%s', returning -1", strVal)); + return -1; + } + } + + private static long getMillisOrNegativeOne(String strVal) + { + try { + return Duration.valueOf(strVal).toMillis(); + } + catch (IllegalArgumentException e) { + log.warn(e, + String.format("Failed to parse io.airlift.units.Duration '%s', returning -1", strVal)); + return -1; + } + } + + private static QueryStageInfo getQueryStageInfo(int stageId, JsonObject stage) + { + QueryStageInfo stageInfo = new QueryStageInfo(); + + stageInfo.stage_id = stageId; + try { + JsonObject stageStats = stage.getJsonObject("stageStats"); + stageInfo.raw_input_data_size_bytes = getBytesOrNegativeOne(stageStats.getString("rawInputDataSize")); + stageInfo.output_data_size_bytes = getBytesOrNegativeOne(stageStats.getString("outputDataSize")); + stageInfo.completed_tasks = stageStats.getInt("completedTasks"); + stageInfo.completed_drivers = stageStats.getInt("completedDrivers"); + stageInfo.cumulative_memory = stageStats.getJsonNumber("cumulativeUserMemory").doubleValue(); + stageInfo.peak_memory_reservation_bytes = getBytesOrNegativeOne(stageStats.getString("peakUserMemoryReservation")); + stageInfo.total_scheduled_time_millis = getMillisOrNegativeOne(stageStats.getString("totalScheduledTime")); + stageInfo.total_cpu_time_millis = getMillisOrNegativeOne(stageStats.getString("totalCpuTime")); + stageInfo.total_user_time_millis = getMillisOrNegativeOne(stageStats.getString("totalUserTime")); + stageInfo.total_blocked_time_millis = getMillisOrNegativeOne(stageStats.getString("totalBlockedTime")); + } + catch (Exception e) { + log.error(e, String.format("Error retrieving stage stats for stage %d", stageId)); + return null; + } + + return stageInfo; + } + + private static OperatorStats getOperatorStat(String operatorSummaryStr) + { + try { + JsonReader jsonReader = Json.createReader(new StringReader(operatorSummaryStr)); + return getOperatorStat(jsonReader.readObject()); + } + catch (Exception e) { + log.error(e, String.format("Error retrieving operator stats from string:\n%s\n", operatorSummaryStr)); + } + + return null; + } + + private static OperatorStats getOperatorStat(JsonObject obj) + { + OperatorStats operatorStats = new OperatorStats(); + + try { + operatorStats.pipeline_id = obj.getInt("pipelineId"); + operatorStats.operator_id = obj.getInt("operatorId"); + operatorStats.plan_node_id = obj.getString("planNodeId"); + operatorStats.operator_type = obj.getString("operatorType"); + operatorStats.total_drivers = obj.getJsonNumber("totalDrivers").longValue(); + operatorStats.add_input_calls = obj.getJsonNumber("addInputCalls").longValue(); + operatorStats.add_input_wall_millis = getMillisOrNegativeOne(obj.getString("addInputWall")); + operatorStats.add_input_cpu_millis = getMillisOrNegativeOne(obj.getString("addInputCpu")); + operatorStats.add_input_user_millis = getMillisOrNegativeOne(obj.getString("addInputUser")); + operatorStats.input_data_size_bytes = getBytesOrNegativeOne(obj.getString("inputDataSize")); + operatorStats.input_positions = obj.getJsonNumber("inputPositions").longValue(); + operatorStats.sum_squared_input_positions = obj.getJsonNumber("sumSquaredInputPositions").doubleValue(); + operatorStats.get_output_calls = obj.getJsonNumber("getOutputCalls").longValue(); + operatorStats.get_output_wall_millis = getMillisOrNegativeOne(obj.getString("getOutputWall")); + operatorStats.get_output_cpu_millis = getMillisOrNegativeOne(obj.getString("getOutputCpu")); + operatorStats.get_output_user_millis = getMillisOrNegativeOne(obj.getString("getOutputUser")); + operatorStats.output_data_size_bytes = getBytesOrNegativeOne(obj.getString("outputDataSize")); + operatorStats.output_positions = obj.getJsonNumber("outputPositions").longValue(); + operatorStats.blocked_wall_millis = getMillisOrNegativeOne(obj.getString("blockedWall")); + operatorStats.finish_calls = obj.getJsonNumber("finishCalls").longValue(); + operatorStats.finish_wall_millis = getMillisOrNegativeOne(obj.getString("finishWall")); + operatorStats.finish_cpu_millis = getMillisOrNegativeOne(obj.getString("finishCpu")); + operatorStats.finish_user_millis = getMillisOrNegativeOne(obj.getString("finishUser")); + operatorStats.memory_reservation_bytes = getBytesOrNegativeOne(obj.getString("userMemoryReservation")); + operatorStats.system_memory_reservation_bytes = getBytesOrNegativeOne(obj.getString("systemMemoryReservation")); + } + catch (Exception e) { + log.error(e, String.format("Error retrieving operator stats from JsonObject:\n%s\n", obj.toString())); + return null; + } + + return operatorStats; + } + + public static Map getQueryStages(QueryMetadata eventMetadata) + { + if (!eventMetadata.getPayload().isPresent()) { + return null; + } + + String payload = eventMetadata.getPayload().get(); + Queue stageJsonObjs = new LinkedList(); + try { + JsonReader jsonReader = Json.createReader(new StringReader(payload)); + stageJsonObjs.add(jsonReader.readObject()); + } + catch (Exception e) { + log.error(e, + String.format("getQueryStages - Unable to extract JsonObject out of following blob:\n%s\n", payload)); + return null; + } + + Map stages = new HashMap(); + while (!stageJsonObjs.isEmpty()) { + JsonObject cur = stageJsonObjs.poll(); + String stageIdStr = "Unknown"; + try { + stageIdStr = cur.getString("stageId"); + int stageId = Integer.parseInt(stageIdStr.split("\\.")[1]); + QueryStageInfo curStage = getQueryStageInfo(stageId, cur); + if (curStage != null) { + stages.put(stageId, getQueryStageInfo(stageId, cur)); + } + } + catch (Exception e) { + log.error(e, + String.format("Failed to parse QueryStageInfo from JsonObject:\n%s\n", cur.toString())); + return null; + } + + try { + cur.getJsonArray("subStages") + .stream() + .filter(val -> val.getValueType() == ValueType.OBJECT) + .forEach(val -> stageJsonObjs.add((JsonObject) val)); + } + catch (Exception e) { + log.error(e, + String.format("Failed to get subStages for stage %s, treating as no subStages", stageIdStr)); + } + } + + return stages; + } + + public static List getOperatorSummaries(QueryStatistics eventStat) + { + try { + return eventStat.getOperatorSummaries() + .stream() + .filter(val -> val != null && !val.isEmpty()) + .map(QueryStatsHelper::getOperatorStat) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + catch (Exception e) { + log.error(e, + String.format("Error converting List to List:\n%s\n", eventStat.getOperatorSummaries().toString())); + } + + return null; + } +} diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListener.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListener.java new file mode 100644 index 0000000000000..6df4eb29c54b5 --- /dev/null +++ b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListener.java @@ -0,0 +1,43 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twitter.presto.plugin.eventlistener; + +import com.facebook.presto.spi.eventlistener.EventListener; +import com.facebook.presto.spi.eventlistener.QueryCompletedEvent; +import com.facebook.presto.spi.eventlistener.QueryCreatedEvent; +import com.facebook.presto.spi.eventlistener.SplitCompletedEvent; +import io.airlift.log.Logger; + +public class TwitterEventListener + implements EventListener +{ + private static final Logger log = Logger.get(TwitterEventListener.class); + private final QueryCompletedEventScriber scriber = new QueryCompletedEventScriber(); + + @Override + public void queryCreated(QueryCreatedEvent queryCreatedEvent) + { + } + + @Override + public void queryCompleted(QueryCompletedEvent queryCompletedEvent) + { + scriber.handle(queryCompletedEvent); + } + + @Override + public void splitCompleted(SplitCompletedEvent splitCompletedEvent) + { + } +} diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListenerFactory.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListenerFactory.java new file mode 100644 index 0000000000000..a22762c9ec32a --- /dev/null +++ b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListenerFactory.java @@ -0,0 +1,35 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twitter.presto.plugin.eventlistener; + +import com.facebook.presto.spi.eventlistener.EventListener; +import com.facebook.presto.spi.eventlistener.EventListenerFactory; + +import java.util.Map; + +public class TwitterEventListenerFactory + implements EventListenerFactory +{ + @Override + public String getName() + { + return "twitter-event-listener"; + } + + @Override + public EventListener create(Map config) + { + return new TwitterEventListener(); + } +} diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListenerPlugin.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListenerPlugin.java new file mode 100644 index 0000000000000..1b77b1decce7f --- /dev/null +++ b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterEventListenerPlugin.java @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twitter.presto.plugin.eventlistener; + +import com.facebook.presto.spi.Plugin; +import com.facebook.presto.spi.eventlistener.EventListenerFactory; +import com.google.common.collect.ImmutableList; + +public class TwitterEventListenerPlugin + implements Plugin +{ + @Override + public Iterable getEventListenerFactories() + { + return ImmutableList.of(new TwitterEventListenerFactory()); + } +} diff --git a/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java new file mode 100644 index 0000000000000..8f4088da6aeb4 --- /dev/null +++ b/twitter-eventlistener-plugin/src/main/java/com/twitter/presto/plugin/eventlistener/TwitterScriber.java @@ -0,0 +1,79 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.twitter.presto.plugin.eventlistener; + +import com.twitter.logging.BareFormatter$; +import com.twitter.logging.Level; +import com.twitter.logging.QueueingHandler; +import com.twitter.logging.ScribeHandler; +import org.apache.thrift.TBase; +import org.apache.thrift.TException; +import org.apache.thrift.TSerializer; + +import java.util.Base64; +import java.util.logging.LogRecord; + +public class TwitterScriber +{ + private static final String DASH = "-"; + private static final int MAX_QUEUE_SIZE = 1000; + + private QueueingHandler queueingHandler; + + // TSerializer is not thread safe + private final ThreadLocal serializer = new ThreadLocal() { + @Override + protected TSerializer initialValue() + { + return new TSerializer(); + } + }; + + public TwitterScriber(String scribeCategory) + { + ScribeHandler scribeHandler = new ScribeHandler( + ScribeHandler.DefaultHostname(), + ScribeHandler.DefaultPort(), + scribeCategory, + ScribeHandler.DefaultBufferTime(), + ScribeHandler.DefaultConnectBackoff(), + ScribeHandler.DefaultMaxMessagesPerTransaction(), + ScribeHandler.DefaultMaxMessagesToBuffer(), + BareFormatter$.MODULE$, + scala.Option.apply((Level) null)); + queueingHandler = new QueueingHandler(scribeHandler, MAX_QUEUE_SIZE); + } + + public void scribe(TBase thriftMessage) + throws TException + { + scribe(serializeThriftToString(thriftMessage)); + } + + /** + * Serialize a thrift object to bytes, compress, then encode as a base64 string. + * Throws TException + */ + private String serializeThriftToString(TBase thriftMessage) + throws TException + { + return Base64.getEncoder().encodeToString(serializer.get().serialize(thriftMessage)); + } + + private void scribe(String message) + { + LogRecord logRecord = new LogRecord(Level.ALL, message); + queueingHandler.publish(logRecord); + } +}