[hotfix] Fix commit bugs of iceberg and improve document (#1770)

Author: luoyuxia

fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java

@@ -609,6 +609,7 @@ private List<SourceSplitBase> getLogSplit(
         return splits;
     }
 
+    /** Return the hybrid lake and fluss splits. Return null if no lake snapshot. */
     @Nullable
     private List<SourceSplitBase> generateHybridLakeFlussSplits() {
         // still have pending lake fluss splits,
@@ -630,9 +631,8 @@ private List<SourceSplitBase> generateHybridLakeFlussSplits() {
             pendingHybridLakeFlussSplits = lakeSplitGenerator.generateHybridLakeFlussSplits();
             return pendingHybridLakeFlussSplits;
         } catch (Exception e) {
-            LOG.error("Failed to get hybrid lake fluss splits, won't take splits in lake.", e);
+            throw new FlinkRuntimeException("Failed to generate hybrid lake fluss splits", e);
         }
-        return null;
     }
 
     private boolean ignoreTableBucket(TableBucket tableBucket) {

fluss-lake/fluss-lake-iceberg/pom.xml

@@ -51,6 +51,13 @@
             <groupId>org.apache.iceberg</groupId>
             <artifactId>iceberg-data</artifactId>
             <version>${iceberg.version}</version>
+            <exclusions>
+                <!-- exclude dependency because of its GPLv2 license, see https://github.com/apache/flink/pull/15599#issuecomment-850241316 -->
+                <exclusion>
+                    <groupId>javax.annotation</groupId>
+                    <artifactId>javax.annotation-api</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.iceberg</groupId>
@@ -297,13 +304,7 @@
                         <configuration>
                             <artifactSet>
                                 <includes>
-                                    <include>org.apache.iceberg:iceberg-api</include>
-                                    <include>org.apache.iceberg:iceberg-core</include>
-                                    <include>org.apache.iceberg:iceberg-data</include>
-                                    <include>org.apache.iceberg:iceberg-common</include>
-                                    <include>org.apache.iceberg:iceberg-parquet</include>
-                                    <include>org.apache.iceberg:iceberg-orc</include>
-                                    <include>org.apache.iceberg:iceberg-bundled-guava</include>
+                                    <include>*:*</include>
                                 </includes>
                             </artifactSet>
                             <filters>

fluss-lake/fluss-lake-iceberg/src/main/java/org/apache/fluss/lake/iceberg/tiering/IcebergLakeCommitter.java

@@ -49,7 +49,6 @@
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -314,8 +313,8 @@ private Snapshot getCommittedLatestSnapshotOfLake(String commitUser) {
         List<Snapshot> snapshots = (List<Snapshot>) icebergTable.snapshots();
         // snapshots() returns snapshots in chronological order (oldest to newest), Reverse to find
         // most recent snapshot committed by Fluss
-        Collections.reverse(snapshots);
-        for (Snapshot snapshot : snapshots) {
+        for (int i = snapshots.size() - 1; i >= 0; i--) {
+            Snapshot snapshot = snapshots.get(i);
             Map<String, String> summary = snapshot.summary();
             if (summary != null && commitUser.equals(summary.get(COMMITTER_USER))) {
                 return snapshot;

fluss-lake/fluss-lake-iceberg/src/main/resources/META-INF/NOTICE

@@ -12,4 +12,42 @@ This project bundles the following dependencies under the Apache Software Licens
 - org.apache.iceberg:iceberg-common:1.9.1
 - org.apache.iceberg:iceberg-data:1.9.1
 - org.apache.iceberg:iceberg-orc:1.9.1
-- org.apache.iceberg:iceberg-parquet:1.9.1
+- org.apache.iceberg:iceberg-parquet:1.9.1
+- org.apache.avro:avro:1.12.0
+- org.apache.parquet:parquet-avro:1.15.2
+- org.apache.parquet:parquet-column:1.15.2
+- org.apache.parquet:parquet-encoding:1.15.2
+- org.apache.parquet:parquet-hadoop:1.15.2
+- org.apache.parquet:parquet-format-structures:1.15.2
+- org.apache.parquet:parquet-jackson:1.15.2
+- org.apache.parquet:parquet-common:1.15.2
+- org.apache.orc:orc-core:nohive:1.9.5
+- org.apache.orc:orc-shims:1.9.5
+- org.apache.commons:commons-lang3:3.18.0
+- org.apache.commons:commons-compress:1.21
+- org.apache.httpcomponents.client5:httpclient5:5.4.3
+- org.apache.httpcomponents.core5:httpcore5:5.3.4
+- org.apache.httpcomponents.core5:httpcore5-h2:5.3.4
+- com.fasterxml.jackson.core:jackson-core:2.15.3
+- com.fasterxml.jackson.core:jackson-databind:2.15.3
+- com.fasterxml.jackson.core:jackson-annotations:2.15.3
+- com.github.ben-manes.caffeine:caffeine:2.9.3
+- org.roaringbitmap:RoaringBitmap:1.3.0
+- io.airlift:aircompressor:0.27
+- org.xerial.snappy:snappy-java:1.1.10.4
+- com.google.code.findbugs:jsr305:1.3.9
+- org.jetbrains:annotations:17.0.0
+- commons-pool:commons-pool:1.6
+- com.google.errorprone:error_prone_annotations:2.10.0
+
+This project bundles the following dependencies under the MIT (https://opensource.org/licenses/MIT)
+See bundled license files for details.
+
+- org.slf4j:slf4j-api:1.7.36
+- org.checkerframework:checker-qual:3.19.0 MIT
+
+This project bundles the following dependencies under BSD License (https://opensource.org/licenses/bsd-license.php).
+See bundled license files for details.
+
+- com.github.luben:zstd-jni:1.5.7-1
+- org.threeten:threeten-extra:1.7.1