add test to flink

Author: JingsongLi

paimon-core/src/main/java/org/apache/paimon/utils/CompressUtils.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.utils;
+
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.zip.GZIPOutputStream;
+
+/** Compress utils. */
+public class CompressUtils {
+
+    public static void gzipCompressFile(String src, String dest) throws IOException {
+        FileInputStream fis = new FileInputStream(src);
+        FileOutputStream fos = new FileOutputStream(dest);
+        GZIPOutputStream gzipOs = new GZIPOutputStream(fos);
+        byte[] buffer = new byte[1024];
+        int bytesRead;
+        while (true) {
+            bytesRead = fis.read(buffer);
+            if (bytesRead == -1) {
+                fis.close();
+                gzipOs.close();
+                return;
+            }
+
+            gzipOs.write(buffer, 0, bytesRead);
+        }
+    }
+}

paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogFormatTableITCaseBase.java

@@ -18,13 +18,19 @@
 
 package org.apache.paimon.hive;
 
+import org.apache.paimon.flink.FormatCatalogTable;
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.Path;
 import org.apache.paimon.hive.runner.PaimonEmbeddedHiveRunner;
+import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.utils.CompressUtils;
 
 import com.klarna.hiverunner.HiveShell;
 import com.klarna.hiverunner.annotations.HiveSQL;
 import org.apache.flink.table.api.EnvironmentSettings;
 import org.apache.flink.table.api.TableEnvironment;
 import org.apache.flink.table.api.internal.TableEnvironmentImpl;
+import org.apache.flink.table.catalog.ObjectPath;
 import org.apache.flink.types.Row;
 import org.apache.flink.util.CloseableIterator;
 import org.junit.Rule;
@@ -40,6 +46,7 @@
 import java.lang.annotation.Target;
 import java.time.LocalDateTime;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -215,6 +222,31 @@ private void doTestJSONFormatTable(String tableName) throws Exception {
                 .containsExactlyInAnyOrder(
                         Row.of(1, LocalDateTime.parse("2025-03-17T10:15:30")),
                         Row.of(2, LocalDateTime.parse("2025-03-18T10:15:30")));
+
+        // test compression
+        FormatCatalogTable catalogTable =
+                (FormatCatalogTable)
+                        tEnv.getCatalog(tEnv.getCurrentCatalog())
+                                .get()
+                                .getTable(new ObjectPath(tEnv.getCurrentDatabase(), tableName));
+        FormatTable table = catalogTable.table();
+        FileIO fileIO = table.fileIO();
+        String file =
+                Arrays.stream(fileIO.listStatus(new Path(table.location())))
+                        .filter(status -> !status.getPath().getName().startsWith("."))
+                        .findFirst()
+                        .get()
+                        .getPath()
+                        .toUri()
+                        .getPath();
+        CompressUtils.gzipCompressFile(file, file + ".gz");
+        fileIO.deleteQuietly(new Path(file));
+
+        assertThat(collect(String.format("SELECT * FROM %s", tableName)))
+                .containsExactlyInAnyOrder(
+                        Row.of(1, LocalDateTime.parse("2025-03-17T10:15:30")),
+                        Row.of(2, LocalDateTime.parse("2025-03-18T10:15:30")));
+
         tEnv.executeSql(
                         String.format(
                                 "INSERT INTO %s VALUES (3, CAST('2025-03-19 10:15:30' AS TIMESTAMP))",

paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/FormatTableTestBase.scala

@@ -22,7 +22,7 @@ import org.apache.paimon.catalog.Identifier
 import org.apache.paimon.fs.Path
 import org.apache.paimon.spark.PaimonHiveTestBase
 import org.apache.paimon.table.FormatTable
-import org.apache.paimon.utils.{FileIOUtils, FileUtils}
+import org.apache.paimon.utils.{CompressUtils, FileIOUtils, FileUtils}
 
 import org.apache.spark.sql.Row
 
@@ -77,28 +77,10 @@ abstract class FormatTableTestBase extends PaimonHiveTestBase {
           .getPath
           .toUri
           .getPath
-        compressGzipFile(file, file + ".gz")
+        CompressUtils.gzipCompressFile(file, file + ".gz")
         fileIO.deleteQuietly(new Path(file))
         checkAnswer(sql("SELECT * FROM compress_t"), Row(1, 2, 3))
       }
     }
   }
-
-  def compressGzipFile(src: String, dest: String): Unit = {
-    val fis = new FileInputStream(src)
-    val fos = new FileOutputStream(dest)
-    val gzipOs = new GZIPOutputStream(fos)
-    val buffer = new Array[Byte](1024)
-    var bytesRead = 0
-    while (true) {
-      bytesRead = fis.read(buffer)
-      if (bytesRead == -1) {
-        fis.close()
-        gzipOs.close()
-        return
-      }
-
-      gzipOs.write(buffer, 0, bytesRead)
-    }
-  }
 }