From 44dbe008389e7c100b236fd7d9e9a5870c6768d2 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sun, 3 Sep 2023 18:34:02 -0700 Subject: [PATCH 1/9] resolves org.apache.iceberg.hive.HiveCatalog ClassNotFoundException --- build.gradle | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.gradle b/build.gradle index da92173..cc5def8 100644 --- a/build.gradle +++ b/build.gradle @@ -37,6 +37,9 @@ dependencies { implementation 'org.xerial:sqlite-jdbc:3.43.0.0' implementation 'org.postgresql:postgresql:42.5.4' + + // Hive Metastore Dependencies + implementation "org.apache.iceberg:iceberg-hive-metastore:${icebergVersion}" } jar { From 23a4dd52ae43c50e6f9fd22ba916dd008ff53927 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sun, 3 Sep 2023 18:34:33 -0700 Subject: [PATCH 2/9] resolves org/apache/thrift/TException NoClassDefFoundError --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index cc5def8..dcc4a9f 100644 --- a/build.gradle +++ b/build.gradle @@ -40,6 +40,7 @@ dependencies { // Hive Metastore Dependencies implementation "org.apache.iceberg:iceberg-hive-metastore:${icebergVersion}" + implementation "org.apache.thrift:libthrift:0.19.0" } jar { From 9186cb4f58b78dee2bfc88debacec5b8089467a8 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sun, 3 Sep 2023 18:51:06 -0700 Subject: [PATCH 3/9] resolves org/apache/hadoop/hive/metastore/api/AlreadyExistsException NoClassDefFoundError --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index dcc4a9f..35ec671 100644 --- a/build.gradle +++ b/build.gradle @@ -41,6 +41,7 @@ dependencies { // Hive Metastore Dependencies implementation "org.apache.iceberg:iceberg-hive-metastore:${icebergVersion}" implementation "org.apache.thrift:libthrift:0.19.0" + implementation "org.apache.hive:hive-metastore:3.1.3" } jar { From 26d8ef152dbbc2b53577cf18d8786bda074c6611 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sun, 3 Sep 2023 19:11:25 -0700 Subject: [PATCH 4/9] resolves org/apache/hadoop/mapred/JobConf NoClassDefFoundError --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index 35ec671..0cc6ea7 100644 --- a/build.gradle +++ b/build.gradle @@ -42,6 +42,7 @@ dependencies { implementation "org.apache.iceberg:iceberg-hive-metastore:${icebergVersion}" implementation "org.apache.thrift:libthrift:0.19.0" implementation "org.apache.hive:hive-metastore:3.1.3" + implementation "org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6" } jar { From 74efec028f65acf0aee62b673d7bde8aa7a34437 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sun, 3 Sep 2023 19:25:45 -0700 Subject: [PATCH 5/9] resolves org.eclipse.jetty.servlet UnsupportedOperationException --- build.gradle | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build.gradle b/build.gradle index 0cc6ea7..132915f 100644 --- a/build.gradle +++ b/build.gradle @@ -43,6 +43,11 @@ dependencies { implementation "org.apache.thrift:libthrift:0.19.0" implementation "org.apache.hive:hive-metastore:3.1.3" implementation "org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6" + constraints { + implementation('org.eclipse.jetty:jetty-servlet:9.4.52.v20230823') { + because 'backwards incompatible changes in earlier versions' + } + } } jar { From 15d6bef902e83928942b87d580f9a73095f35c92 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sun, 3 Sep 2023 20:08:06 -0700 Subject: [PATCH 6/9] resolves org/apache/thrift/transport/TFramedTransport NoClassDefFoundError --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 132915f..ac34d7f 100644 --- a/build.gradle +++ b/build.gradle @@ -40,7 +40,7 @@ dependencies { // Hive Metastore Dependencies implementation "org.apache.iceberg:iceberg-hive-metastore:${icebergVersion}" - implementation "org.apache.thrift:libthrift:0.19.0" + implementation "org.apache.thrift:libthrift:0.9.3" implementation "org.apache.hive:hive-metastore:3.1.3" implementation "org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6" constraints { From 10304071bf2b5c3eae2aacc1e953773a5abbeec2 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sun, 3 Sep 2023 20:27:31 -0700 Subject: [PATCH 7/9] standardize hadoop version --- build.gradle | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index ac34d7f..c7cd7ef 100644 --- a/build.gradle +++ b/build.gradle @@ -10,6 +10,7 @@ repositories { ext { icebergVersion = '1.3.1' awsSdkVersion = '2.20.18' + hadoopVersion = '3.3.4' } dependencies { @@ -19,8 +20,8 @@ dependencies { implementation "org.apache.iceberg:iceberg-aws:${icebergVersion}" implementation "org.apache.iceberg:iceberg-bundled-guava:${icebergVersion}" - implementation 'org.apache.hadoop:hadoop-common:3.3.4' - implementation 'org.apache.hadoop:hadoop-hdfs-client:3.3.4' + implementation "org.apache.hadoop:hadoop-common:${hadoopVersion}" + implementation "org.apache.hadoop:hadoop-hdfs-client:${hadoopVersion}" runtimeOnly "software.amazon.awssdk:url-connection-client:${awsSdkVersion}" runtimeOnly "software.amazon.awssdk:apache-client:${awsSdkVersion}" @@ -42,7 +43,7 @@ dependencies { implementation "org.apache.iceberg:iceberg-hive-metastore:${icebergVersion}" implementation "org.apache.thrift:libthrift:0.9.3" implementation "org.apache.hive:hive-metastore:3.1.3" - implementation "org.apache.hadoop:hadoop-mapreduce-client-core:3.3.6" + implementation "org.apache.hadoop:hadoop-mapreduce-client-core:${hadoopVersion}" constraints { implementation('org.eclipse.jetty:jetty-servlet:9.4.52.v20230823') { because 'backwards incompatible changes in earlier versions' From 079eaae636aeb0a84c329d7cdf4f122e761a3de5 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sun, 3 Sep 2023 20:33:03 -0700 Subject: [PATCH 8/9] create docker-compose yml --- docker-compose.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..ee83ef8 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,9 @@ +services: + iceberg-rest-catalog: + build: . + image: tabulario/iceberg-rest + ports: + - 8181:8181 + environment: + CATALOG_CATALOG__IMPL: org.apache.iceberg.hive.HiveCatalog + CATALOG_URI: thrift://host.docker.internal:9083 From 6b50de9ee030d5834dd4f64f31d8511963f2a4f7 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Tue, 5 Sep 2023 10:59:19 -0700 Subject: [PATCH 9/9] libthrift is imported by hive-metastore:3.1.3 --- build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/build.gradle b/build.gradle index c7cd7ef..1e294f7 100644 --- a/build.gradle +++ b/build.gradle @@ -41,7 +41,6 @@ dependencies { // Hive Metastore Dependencies implementation "org.apache.iceberg:iceberg-hive-metastore:${icebergVersion}" - implementation "org.apache.thrift:libthrift:0.9.3" implementation "org.apache.hive:hive-metastore:3.1.3" implementation "org.apache.hadoop:hadoop-mapreduce-client-core:${hadoopVersion}" constraints {