add paimon hadoop remote storage

Author: beryllw

website/docs/quickstart/flink.md

@@ -37,12 +37,55 @@ cd fluss-quickstart-flink
 
 ```yaml
 services:
+  zookeeper:
+    restart: always
+    image: zookeeper:3.9.2
+  namenode:
+    image: apache/hadoop:3.3.6
+    hostname: namenode
+    user: root
+    command: [ "hdfs", "namenode" ]
+    ports:
+      - 9870:9870
+      - 8020:8020
+    environment:
+      ENSURE_NAMENODE_DIR: "/tmp/hadoop/dfs/name"
+      CORE-SITE.XML_fs.defaultFS: hdfs://namenode:8020
+      CORE-SITE.XML_hadoop.tmp.dir: /hadoop/tmp
+      HDFS-SITE.XML_dfs.namenode.rpc-address: namenode:8020
+      HDFS-SITE.XML_dfs.replication: 1
+      HDFS-SITE.XML_dfs.permissions.enabled: false
+      HDFS-SITE.XML_dfs.datanode.address: datanode:9866
+    healthcheck:
+      test: ["CMD", "hdfs dfs -ls /"]
+      interval: 10s
+      timeout: 10s
+      retries: 20
+
+  datanode:
+    image: apache/hadoop:3.3.6
+    user: root
+    command: [ "hdfs", "datanode" ]
+    environment:
+      CORE-SITE.XML_fs.defaultFS: hdfs://namenode:8020
+      CORE-SITE.XML_hadoop.tmp.dir: /hadoop/tmp
+      HDFS-SITE.XML_dfs.namenode.rpc-address: namenode:8020
+      HDFS-SITE.XML_dfs.replication: 1
+      HDFS-SITE.XML_dfs.permissions.enabled: false
+      HDFS-SITE.XML_dfs.datanode.address: datanode:9866
+    depends_on:
+      - namenode
+  
   #begin Fluss cluster
   coordinator-server:
     image: fluss/fluss:$FLUSS_DOCKER_VERSION$
-    command: coordinatorServer
     depends_on:
-      - zookeeper
+      namenode:
+        condition: service_healthy
+      zookeeper:
+        condition: service_started
+      datanode:
+        condition: service_started
     environment:
       - |
         FLUSS_PROPERTIES=
@@ -51,9 +94,10 @@ services:
         remote.data.dir: /tmp/fluss/remote-data
         datalake.format: paimon
         datalake.paimon.metastore: filesystem
-        datalake.paimon.warehouse: /tmp/paimon
+        datalake.paimon.warehouse: hdfs://namenode:8020/fluss-lake
     volumes:
-      - shared-tmpfs:/tmp/paimon
+      - ./lib:/tmp/lib
+    entrypoint: [ "sh", "-c", "cp -v /tmp/lib/*.jar /opt/fluss/plugins/iceberg/ && exec /docker-entrypoint.sh coordinatorServer" ]
   tablet-server:
     image: fluss/fluss:$FLUSS_DOCKER_VERSION$
     command: tabletServer
@@ -66,15 +110,11 @@ services:
         bind.listeners: FLUSS://tablet-server:9123
         data.dir: /tmp/fluss/data
         remote.data.dir: /tmp/fluss/remote-data
-        kv.snapshot.interval: 0s
+        kv.snapshot.interval: 30s
         datalake.format: paimon
         datalake.paimon.metastore: filesystem
-        datalake.paimon.warehouse: /tmp/paimon
-    volumes:
-      - shared-tmpfs:/tmp/paimon
-  zookeeper:
-    restart: always
-    image: zookeeper:3.9.2
+        datalake.paimon.warehouse: hdfs://namenode:8020/fluss-lake
+
   #end
   #begin Flink cluster
   jobmanager:
@@ -86,8 +126,6 @@ services:
       - |
         FLINK_PROPERTIES=
         jobmanager.rpc.address: jobmanager
-    volumes:
-      - shared-tmpfs:/tmp/paimon
   taskmanager:
     image: fluss/quickstart-flink:1.20-$FLUSS_DOCKER_VERSION$
     depends_on:
@@ -100,16 +138,7 @@ services:
         taskmanager.numberOfTaskSlots: 10
         taskmanager.memory.process.size: 2048m
         taskmanager.memory.framework.off-heap.size: 256m
-    volumes:
-      - shared-tmpfs:/tmp/paimon
   #end
-  
-volumes:
-  shared-tmpfs:
-    driver: local
-    driver_opts:
-      type: "tmpfs"
-      device: "tmpfs"
 ```
 
 The Docker Compose environment consists of the following containers:
@@ -346,6 +375,33 @@ The following SQL query should return an empty result.
 SELECT * FROM fluss_customer WHERE `cust_key` = 1;
 ```
 
+## Fluss Remote Storage
+
+Finally, you can use the following command to view the fluss kv snapshot stored in fluss remote storage:
+```shell
+docker compose exec namenode hdfs dfs -ls -R /fluss-data/ | awk '{print $8}' | grep -v '^$' | tree --fromfile .
+```
+
+**Sample Output:**
+```shell
+hdfs://namenode:8020/fluss-data
+└── kv
+    └── fluss
+        ├── enriched_orders-3
+        │   └── 0
+        │       ├── shared
+        │       │   ├── 71fca534-ecca-489b-a19a-bd0538c9f9e9
+        │       │   ├── b06ef3a3-2873-470e-961f-da25582136a1
+        │       │   └── b93bad5c-00fb-4e62-8217-71b010621479
+        │       └── snap-2
+        │           ├── _METADATA
+        │           ├── 08d39726-f847-4401-8f31-4e905f2ba3f6
+        │           ├── b6a7bc2c-b5c3-4eeb-a523-b2b6fff159f3
+        │           └── e6278555-d71f-431f-954e-71bf066dd29f
+        ├── fluss_customer-1
+        ... # Remaining entries omitted for brevity
+```
+
 ## Integrate with Paimon
 ### Start the Lakehouse Tiering Service
 To integrate with [Apache Paimon](https://paimon.apache.org/), you need to start the `Lakehouse Tiering Service`. 
@@ -473,30 +529,30 @@ The result looks like:
 ```
 You can execute the real-time analytics query multiple times, and the results will vary with each run as new data is continuously written to Fluss in real-time.
 
-Finally, you can use the following command to view the files stored in Paimon:
+### Storage
+
+Finally, you can use the following command to view the files stored in Paimon Hadoop warehouse:
 ```shell
-docker compose exec taskmanager tree /tmp/paimon/fluss.db
+docker compose exec namenode hdfs dfs -ls -R /fluss-lake/ | awk '{print $8}' | grep -v '^$' | tree --fromfile .
 ```
 
 **Sample Output:**
 ```shell
-/tmp/paimon/fluss.db
-└── datalake_enriched_orders
-    ├── bucket-0
-    │   ├── changelog-aef1810f-85b2-4eba-8eb8-9b136dec5bdb-0.orc
-    │   └── data-aef1810f-85b2-4eba-8eb8-9b136dec5bdb-1.orc
-    ├── manifest
-    │   ├── manifest-aaa007e1-81a2-40b3-ba1f-9df4528bc402-0
-    │   ├── manifest-aaa007e1-81a2-40b3-ba1f-9df4528bc402-1
-    │   ├── manifest-list-ceb77e1f-7d17-4160-9e1f-f334918c6e0d-0
-    │   ├── manifest-list-ceb77e1f-7d17-4160-9e1f-f334918c6e0d-1
-    │   └── manifest-list-ceb77e1f-7d17-4160-9e1f-f334918c6e0d-2
-    ├── schema
-    │   └── schema-0
-    └── snapshot
-        ├── EARLIEST
-        ├── LATEST
-        └── snapshot-1
+hdfs://namenode:8020/fluss-lake
+├── default.db
+└── fluss.db
+    └── datalake_enriched_orders
+        ├── bucket-0
+        │   └── data-02acf76d-c4cc-4bc1-9292-e64a77dfcc72-0.parquet
+        ├── manifest
+        │   ├── manifest-df5b6833-7e92-4ec9-a196-51d6fd60b1d1-0
+        │   ├── manifest-list-b683c5a2-4072-4c7a-8586-2c853de8d964-0
+        │   └── manifest-list-b683c5a2-4072-4c7a-8586-2c853de8d964-1
+        ├── schema
+        │   └── schema-0
+        └── snapshot
+            ├── LATEST
+            └── snapshot-1
 ```
 The files adhere to Paimon's standard format, enabling seamless querying with other engines such as [StarRocks](https://docs.starrocks.io/docs/data_source/catalog/paimon_catalog/).