feat: logs API

.github/workflows/docker-images.yml

@@ -17,7 +17,7 @@
 # Modified by Datazip Inc. in 2026
 
 
-# Build and publish Docker images. Triggered by push (dev-v3) or workflow_call with tag (e.g. release).
+# Build and publish Docker images. Triggered by push (dev-v5) or workflow_call with tag (e.g. release).
 
 name: Publish Docker Image
 
@@ -66,7 +66,7 @@ jobs:
       - name: Set Docker tags
         id: meta
         run: |
-          VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-v3' }}"
+          VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-v5' }}"
           if [ "${{ github.ref }}" = "refs/heads/master" ]; then
             case "${VERSION_TAG}" in v*) ;; *) echo "::error::On master branch, version tag must start with 'v' (e.g. v1.0.0). Got: ${VERSION_TAG}"; exit 1 ;; esac
             echo "tags=olakego/fusion:latest,olakego/fusion:${VERSION_TAG}" >> $GITHUB_OUTPUT
@@ -122,7 +122,7 @@ jobs:
       - name: Set Docker tags
         id: meta
         run: |
-          VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-v3' }}"
+          VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-v5' }}"
           if [ "${{ github.ref }}" = "refs/heads/master" ]; then
             case "${VERSION_TAG}" in v*) ;; *) echo "::error::On master branch, version tag must start with 'v' (e.g. v1.0.0). Got: ${VERSION_TAG}"; exit 1 ;; esac
             echo "tags=olakego/fusion-spark:latest,olakego/fusion-spark:${VERSION_TAG}" >> $GITHUB_OUTPUT

.gitignore

@@ -39,6 +39,7 @@ spark/spark-warehouse/
 
 # vscode/eclipse files
 .vscode/
+.vscode/
 .classpath
 .project
 bin/

.vscode/launch.json

@@ -32,7 +32,7 @@
       "vmArgs": "-Dfile.encoding=UTF-8 -Darrow.memory.allocator=unsafe -XX:+UseZGC -XX:CompileCommand=exclude,io/netty/buffer/PoolChunkList.allocate -XX:CompileCommand=exclude,io/netty/buffer/PoolArena.allocate --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED",
       "env": {
         "AMORO_HOME": "${workspaceFolder}/dist/src/main/amoro-bin",
-        "AMORO_CONF_DIR": "${workspaceFolder}/docker/kind",
+        "AMORO_CONF_DIR": "${workspaceFolder}/local-test",
         "CONSOLE_LOG_LEVEL": "info",
         "OPTIMIZER_JAVA_OPTS": "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5006",
         "AMS_SERVER__EXPOSE__HOST": "127.0.0.1",

Makefile

@@ -18,6 +18,7 @@
 
 COMPOSE_DIR  := local-test
 KIND_CLUSTER := fusion-cluster
+AMORO_LOG_DIR ?= $(CURDIR)/docker/kind/amoro-logs
 DIST_TAR     := $(CURDIR)/dist/target/apache-amoro-0.9-SNAPSHOT-bin.tar.gz
 RUNTIME_HOME := $(CURDIR)/dist/target/amoro-0.9-SNAPSHOT
 BIN_HOME     := $(CURDIR)/dist/src/main/amoro-bin
@@ -118,7 +119,8 @@ stop-deps:
 
 start-fusion-docker:
 	@echo "Starting Fusion (Kind cluster + all services)..."
-	@docker compose -f $(COMPOSE_DIR)/docker-compose.yml --profile prod up -d
+	@mkdir -p "$(AMORO_LOG_DIR)"
+	@AMORO_LOG_DIR="$(AMORO_LOG_DIR)" docker compose -f $(COMPOSE_DIR)/docker-compose.yml --profile prod up -d
 	@kind export kubeconfig --name $(KIND_CLUSTER) 2>/dev/null
 
 clean-fusion-docker:

amoro-ams/pom.xml

@@ -455,6 +455,11 @@
             <version>${pagehelper.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.iceberg</groupId>
             <artifactId>iceberg-data</artifactId>

amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java

@@ -14,6 +14,8 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
+ * Modified by Datazip Inc. in 2026
  */
 
 package org.apache.amoro.server;
@@ -95,7 +97,7 @@ public class AmoroManagementConf {
   public static final ConfigOption<Duration> REFRESH_EXTERNAL_CATALOGS_INTERVAL =
       ConfigOptions.key("refresh-external-catalogs.interval")
           .durationType()
-          .defaultValue(Duration.ofMinutes(3))
+          .defaultValue(Duration.ofMinutes(1))
           .withDescription("Interval to refresh the external catalog.");
 
   public static final ConfigOption<Integer> REFRESH_EXTERNAL_CATALOGS_THREAD_COUNT =
@@ -111,11 +113,12 @@ public class AmoroManagementConf {
           .defaultValue(1000000)
           .withDescription("The queue size of the executors of the external catalog explorer.");
 
-  public static final ConfigOption<Boolean> EXPIRE_SNAPSHOTS_ENABLED =
+  public static final ConfigOption<Boolean>
+      EXPIRE_SNAPSHOTS_ENABLED = // TODO: will be enabled with cleaning feature
       ConfigOptions.key("expire-snapshots.enabled")
-          .booleanType()
-          .defaultValue(true)
-          .withDescription("Enable snapshots expiring.");
+              .booleanType()
+              .defaultValue(false)
+              .withDescription("Enable snapshots expiring.");
 
   public static final ConfigOption<Integer> EXPIRE_SNAPSHOTS_THREAD_COUNT =
       ConfigOptions.key("expire-snapshots.thread-count")
@@ -132,7 +135,7 @@ public class AmoroManagementConf {
   public static final ConfigOption<Boolean> CLEAN_ORPHAN_FILES_ENABLED =
       ConfigOptions.key("clean-orphan-files.enabled")
           .booleanType()
-          .defaultValue(true)
+          .defaultValue(false) // TODO: will be enabled with cleaning feature
           .withDescription("Enable orphan files cleaning.");
 
   public static final ConfigOption<Integer> CLEAN_ORPHAN_FILES_THREAD_COUNT =
@@ -150,7 +153,7 @@ public class AmoroManagementConf {
   public static final ConfigOption<Boolean> CLEAN_DANGLING_DELETE_FILES_ENABLED =
       ConfigOptions.key("clean-dangling-delete-files.enabled")
           .booleanType()
-          .defaultValue(true)
+          .defaultValue(false) // TODO: will be enabled with cleaning feature
           .withDescription("Enable dangling delete files cleaning.");
 
   public static final ConfigOption<Integer> CLEAN_DANGLING_DELETE_FILES_THREAD_COUNT =
@@ -186,7 +189,7 @@ public class AmoroManagementConf {
   public static final ConfigOption<Boolean> AUTO_CREATE_TAGS_ENABLED =
       ConfigOptions.key("auto-create-tags.enabled")
           .booleanType()
-          .defaultValue(true)
+          .defaultValue(false) // TODO: will be enabled with cleaning feature
           .withDescription("Enable creating tags.");
 
   public static final ConfigOption<Integer> AUTO_CREATE_TAGS_THREAD_COUNT =

amoro-ams/src/main/java/org/apache/amoro/server/DefaultOptimizingService.java

@@ -14,6 +14,8 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
+ * Modified by Datazip Inc. in 2026
  */
 
 package org.apache.amoro.server;
@@ -369,7 +371,12 @@ private class TableRuntimeHandlerImpl extends RuntimeHandlerChain {
     @Override
     public void handleStatusChanged(TableRuntime tableRuntime, OptimizingStatus originalStatus) {
       DefaultTableRuntime defaultTableRuntime = (DefaultTableRuntime) tableRuntime;
-      if (!defaultTableRuntime.getOptimizingStatus().isProcessing()) {
+      OptimizingStatus newStatus = defaultTableRuntime.getOptimizingStatus();
+      // Only re-queue when the table becomes available for the next planning cycle (IDLE after
+      // completing/skipping work) or has new pending data (PENDING). Skip the IDLE→PLANNING
+      // transition to avoid a redundant "Bind queue" log and unnecessary scheduler churn while
+      // planning is already in flight.
+      if (newStatus == OptimizingStatus.IDLE || newStatus == OptimizingStatus.PENDING) {
         getOptionalQueueByGroup(defaultTableRuntime.getGroupName())
             .ifPresent(q -> q.refreshTable(defaultTableRuntime));
       }

amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java

@@ -14,6 +14,8 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
+ *
+ * Modified by Datazip Inc. in 2026
  */
 
 package org.apache.amoro.server.dashboard;
@@ -43,6 +45,7 @@
 import org.apache.amoro.server.dashboard.controller.ApiTokenController;
 import org.apache.amoro.server.dashboard.controller.CatalogController;
 import org.apache.amoro.server.dashboard.controller.HealthCheckController;
+import org.apache.amoro.server.dashboard.controller.LogController;
 import org.apache.amoro.server.dashboard.controller.LoginController;
 import org.apache.amoro.server.dashboard.controller.OptimizerController;
 import org.apache.amoro.server.dashboard.controller.OptimizerGroupController;
@@ -90,6 +93,7 @@ public class DashboardServer {
   private final VersionController versionController;
   private final OverviewController overviewController;
   private final ApiTokenController apiTokenController;
+  private final LogController logController;
 
   private final PasswdAuthenticationProvider basicAuthProvider;
   private final TokenAuthenticationProvider jwtAuthProvider;
@@ -120,6 +124,7 @@ public DashboardServer(
     this.overviewController = new OverviewController(manager);
     APITokenManager apiTokenManager = new APITokenManager();
     this.apiTokenController = new ApiTokenController(apiTokenManager);
+    this.logController = new LogController();
 
     String authType = serviceConfig.get(AmoroManagementConf.HTTP_SERVER_REST_AUTH_TYPE);
     this.basicAuthProvider =
@@ -138,6 +143,7 @@ public DashboardServer(
   }
 
   private volatile String indexHtml = null;
+
   // read index.html content
   public String getIndexFileContent() {
     if (indexHtml == null) {
@@ -396,6 +402,15 @@ private EndpointGroup apiGroup() {
             post("/calculate/signature", apiTokenController::calculateSignature);
             post("/calculate/encryptString", apiTokenController::getEncryptStringFromQueryParam);
           });
+
+      // logs api
+      path(
+          "/logs",
+          () -> {
+            get("/process/{processId}", logController::getProcessLogs);
+            get("/process/{processId}/download", logController::downloadProcessLogs);
+            get("/process/{processId}/file/{fileId}", logController::downloadLogFile);
+          });
     };
   }