From 565bc645fc90ffc0e387fa3d5cfc50665928ba93 Mon Sep 17 00:00:00 2001 From: hashcode-ankit Date: Wed, 11 Feb 2026 22:50:45 +0530 Subject: [PATCH 01/46] feat: adding fusion releasor and kind cluster setup --- .github/workflows/docker-images.yml | 176 +++------- .github/workflows/draft-changelog.yml | 54 +++ .../workflows/prevent-direct-master-pr.yml | 40 +++ .github/workflows/release-approval.yml | 19 ++ .github/workflows/todo.yml | 41 +++ .gitignore | 1 + Makefile | 87 +++++ docker/kind/config.yaml | 175 ++++++++++ docker/kind/docker-compose.yml | 316 ++++++++++++++++++ docker/kind/kind-config.yaml | 40 +++ docker/kind/spark-rbac.yaml | 76 +++++ 11 files changed, 889 insertions(+), 136 deletions(-) create mode 100644 .github/workflows/draft-changelog.yml create mode 100644 .github/workflows/prevent-direct-master-pr.yml create mode 100644 .github/workflows/release-approval.yml create mode 100644 .github/workflows/todo.yml create mode 100644 Makefile create mode 100644 docker/kind/config.yaml create mode 100644 docker/kind/docker-compose.yml create mode 100644 docker/kind/kind-config.yaml create mode 100644 docker/kind/spark-rbac.yaml diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index 2f7c47bf30..f772a5d4b6 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -14,19 +14,20 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# Modified by Olake by Datazip Inc. in 2026 -# This workflow will build docker images when commit merged or pushed to master. -# or tags pushed. +# Build and publish Docker images. Triggered by push (dev-latest) or workflow_call with tag (e.g. release). name: Publish Docker Image on: - push: - branches: - - "master" - tags: - - "v*" + workflow_call: + inputs: + tag: + description: 'Version tag for the image (e.g. v1.0.0).' + type: string + required: true concurrency: @@ -35,12 +36,10 @@ concurrency: jobs: docker-amoro: - name: Push Amoro Docker Image to Docker Hub + name: Push Fusion Docker Image runs-on: ubuntu-latest - if: ${{ startsWith(github.repository, 'apache/') }} - strategy: - matrix: - hadoop: [ "v2", "v3" ] + environment: docker publish + if: ${{ startsWith(github.repository, 'datazip-inc/') }} steps: - uses: actions/checkout@v3 - name: Set up JDK 11 @@ -56,21 +55,16 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Set up Docker tags - uses: docker/metadata-action@v5 + - name: Set Docker tags id: meta - with: - flavor: | - latest=false - images: | - name=apache/amoro - tags: | - type=ref,event=branch,enable=${{ matrix.hadoop == 'v3' }},suffix=-snapshot - type=ref,event=branch,enable=${{ matrix.hadoop == 'v2' }},suffix=-snapshot-hadoop2 - type=raw,enable=${{ matrix.hadoop == 'v3' && startsWith(github.ref, 'refs/tags/v') }},value=latest - type=semver,event=tag,enable=${{ matrix.hadoop == 'v3' }},pattern={{version}} - type=semver,event=tag,enable=${{ matrix.hadoop == 'v3' }},pattern={{version}}, suffix=-hadoop3 - type=semver,event=tag,enable=${{ matrix.hadoop == 'v2' }},pattern={{version}}, suffix=-hadoop2 + run: | + VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-latest' }}" + if [ "${{ github.ref }}" = "refs/heads/master" ]; then + case "${VERSION_TAG}" in v*) ;; *) echo "::error::On master branch, version tag must start with 'v' (e.g. v1.0.0). Got: ${VERSION_TAG}"; exit 1 ;; esac + echo "tags=olakego/fusion:latest,olakego/fusion:${VERSION_TAG}" >> $GITHUB_OUTPUT + else + echo "tags=olakego/fusion:latest-${VERSION_TAG},olakego/fusion:${VERSION_TAG}" >> $GITHUB_OUTPUT + fi - name: Print tags run: echo '${{ steps.meta.outputs.tags }}' @@ -78,18 +72,13 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Set Maven Build Properties - if: ${{ matrix.hadoop == 'v2' }} - run: | - echo "MVN_HADOOP=-Phadoop2" >> $GITHUB_ENV + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} - name: Build dist module with Maven - run: ./mvnw clean package -pl 'dist' -am -e ${MVN_HADOOP} -DskipTests -B -ntp -Psupport-all-formats -Pno-extended-disk-storage -Pno-plugin-bin + run: ./mvnw clean package -pl 'dist' -am -e -DskipTests -B -ntp -Psupport-all-formats -Pno-extended-disk-storage -Pno-plugin-bin - - name: Build and Push Amoro Docker Image + - name: Build and Push Fusion Docker Image to olakego/fusion uses: docker/build-push-action@v4 with: context: . @@ -98,13 +87,15 @@ jobs: platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} - docker-optimizer-flink: - name: Push Amoro Optimizer-Flink Docker Image to Docker Hub + docker-optimizer-spark: + name: Push Fusion Spark Optimizer Docker Image to olakego/fusion-spark runs-on: ubuntu-latest - if: ${{ startsWith(github.repository, 'apache/') }} + environment: docker publish + if: ${{ startsWith(github.repository, 'datazip-inc/') }} strategy: matrix: - flink: [ "1.14.6", "1.20.0" ] + spark: [ "3.5.7" ] # spark version supported + scala: [ "2.12.15" ] # scala version supported steps: - uses: actions/checkout@v3 - name: Set up JDK 11 @@ -120,112 +111,25 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Set up Docker tags - uses: docker/metadata-action@v5 + - name: Set Docker tags id: meta - with: - flavor: | - latest=false - images: | - name=apache/amoro-flink-optimizer - tags: | - type=ref,event=branch,enable=${{ matrix.flink == '1.14.6' }},suffix=-snapshot - type=ref,event=branch,enable=${{ matrix.flink == '1.14.6' }},suffix=-snapshot-flink1.14 - type=ref,event=branch,enable=${{ matrix.flink == '1.20.0' }},suffix=-snapshot-flink1.20 - type=raw,enable=${{ matrix.hadoop == '1.14.6' && startsWith(github.ref, 'refs/tags/v') }},value=latest - type=semver,enable=${{ matrix.flink == '1.14.6' }},pattern={{version}} - type=semver,enable=${{ matrix.flink == '1.14.6' }},pattern={{version}}, suffix=-flink1.14 - type=semver,enable=${{ matrix.flink == '1.20.0' }},pattern={{version}}, suffix=-flink1.20 - - - name: Print tags - run: echo '${{ steps.meta.outputs.tags }}' - - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Set optimizer flink version run: | - OPTIMIZER_FLINK=${{ matrix.flink }} && \ - echo "OPTIMIZER_FLINK=-Dflink-optimizer.flink-version${OPTIMIZER_FLINK}" >> $GITHUB_ENV - if [[ "$OPTIMIZER_FLINK" < "1.15" ]]; then - echo "Adding -Pflink-optimizer-pre-1.15 for Flink version less than 1.15" - echo "OPTIMIZER_FLINK=-Pflink-optimizer-pre-1.15 -Dflink-optimizer.flink-version=${OPTIMIZER_FLINK}" >> $GITHUB_ENV + VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-latest' }}" + if [ "${{ github.ref }}" = "refs/heads/master" ]; then + case "${VERSION_TAG}" in v*) ;; *) echo "::error::On master branch, version tag must start with 'v' (e.g. v1.0.0). Got: ${VERSION_TAG}"; exit 1 ;; esac + echo "tags=olakego/fusion-spark:latest,olakego/fusion-spark:${VERSION_TAG}" >> $GITHUB_OUTPUT + else + echo "tags=olakego/fusion-spark:latest-${VERSION_TAG},olakego/fusion-spark:${VERSION_TAG}" >> $GITHUB_OUTPUT fi - - name: Set ENV Amoro version - id: version - run: | - AMORO_VERSION=`cat pom.xml | grep 'amoro-parent' -C 3 | grep -Eo '.*' | awk -F'[><]' '{print $3}'` \ - && echo "$AMORO_VERSION" \ - && echo "AMORO_VERSION=${AMORO_VERSION}" >> $GITHUB_ENV \ - && echo "AMORO_VERSION=${AMORO_VERSION}" >> $GITHUB_OUTPUT - - - name: Build optimizer module with Maven - run: ./mvnw clean package -pl 'amoro-optimizer/amoro-optimizer-flink' -am -e ${OPTIMIZER_FLINK} -DskipTests -B -ntp - - - name: Build and Push Flink Optimizer Docker Image - uses: docker/build-push-action@v4 - env: - AMORO_VERSION: ${{ steps.version.outputs.AMORO_VERSION }} - with: - context: . - push: true - file: docker/optimizer-flink/Dockerfile - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.meta.outputs.tags }} - build-args: | - FLINK_VERSION=${{ matrix.flink }} - OPTIMIZER_JOB=amoro-optimizer/amoro-optimizer-flink/target/amoro-optimizer-flink-${{ env.AMORO_VERSION }}-jar-with-dependencies.jar - - docker-optimizer-spark: - name: Push Amoro Optimizer-Spark Docker Image to Docker Hub - runs-on: ubuntu-latest - if: ${{ startsWith(github.repository, 'apache/') }} - strategy: - matrix: - spark: [ "3.5.7" ] - scala: [ "2.12.15" ] - steps: - - uses: actions/checkout@v3 - - name: Set up JDK 11 - uses: actions/setup-java@v3 - with: - java-version: '11' - distribution: 'temurin' - cache: maven - check-latest: false - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Set up Docker tags - uses: docker/metadata-action@v5 - id: meta - with: - flavor: | - latest=false - images: | - name=apache/amoro-spark-optimizer - tags: | - type=ref,event=branch,enable=${{ matrix.spark == '3.5.7' }},suffix=-snapshot - type=ref,event=branch,enable=${{ matrix.spark == '3.5.7' }},suffix=-snapshot-spark3.5 - type=raw,enable=${{ matrix.hadoop == '3.5.7' && startsWith(github.ref, 'refs/tags/v') }},value=latest - type=semver,enable=${{ matrix.spark == '3.5.7' }},pattern={{version}} - type=semver,enable=${{ matrix.spark == '3.5.7' }},pattern={{version}}, suffix=-spark3.5 - - name: Print tags run: echo '${{ steps.meta.outputs.tags }}' - name: Login to Docker Hub uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} - name: Set optimizer spark version and extract versions id: versions @@ -247,7 +151,7 @@ jobs: - name: Build optimizer module with Maven run: ./mvnw clean package -pl 'amoro-optimizer/amoro-optimizer-spark' -am -e ${SPARK_VERSION} -DskipTests -B -ntp - - name: Build and Push Spark Optimizer Docker Image + - name: Build and Push Spark Optimizer Docker Image to olakego/fusion-spark uses: docker/build-push-action@v4 env: AMORO_VERSION: ${{ steps.version.outputs.AMORO_VERSION }} diff --git a/.github/workflows/draft-changelog.yml b/.github/workflows/draft-changelog.yml new file mode 100644 index 0000000000..48a0bc4e87 --- /dev/null +++ b/.github/workflows/draft-changelog.yml @@ -0,0 +1,54 @@ +name: Draft Releaser From Master + +on: + pull_request: + types: [closed] + branches: + - master + +jobs: + create_draft_release: + if: github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'master' && github.event.pull_request.head.ref == 'staging' + name: Create Draft Release + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get latest release + id: latest-release + run: | + latest_tag=$(git tag -l | sort -V | tail -1) + echo "LATEST_TAG=$latest_tag" >> $GITHUB_ENV + + - name: Generate next version + id: generate-next-version + run: | + if [ -z "$LATEST_TAG" ]; then + next_version="v0.0.0" + else + # Remove 'v' prefix and split version into array + version=${LATEST_TAG#v} + IFS='.' read -ra VERSION_PARTS <<< "$version" + + # Increment the last number (patch version) + VERSION_PARTS[2]=$((VERSION_PARTS[2] + 1)) + + # Reconstruct the version with 'v' prefix + next_version="v${VERSION_PARTS[0]}.${VERSION_PARTS[1]}.${VERSION_PARTS[2]}" + fi + echo "NEXT_VERSION=$next_version" >> $GITHUB_ENV + + - name: Create draft release + id: create-draft-release + uses: ncipollo/release-action@v1 + with: + tag: ${{ env.NEXT_VERSION }} + generateReleaseNotes: true + draft: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/prevent-direct-master-pr.yml b/.github/workflows/prevent-direct-master-pr.yml new file mode 100644 index 0000000000..c34b8ab700 --- /dev/null +++ b/.github/workflows/prevent-direct-master-pr.yml @@ -0,0 +1,40 @@ +name: Master Branch Protection + +on: + pull_request: + types: [opened, edited, reopened, synchronize] + +jobs: + enforce-rules: + runs-on: ubuntu-latest + steps: + - name: Validate Branch Rules + run: | + SOURCE_BRANCH="${{ github.event.pull_request.head.ref }}" + TARGET_BRANCH="${{ github.event.pull_request.base.ref }}" + + # Allow only staging → master PRs + if [ "$TARGET_BRANCH" == "master" ]; then + if [ "$SOURCE_BRANCH" != "staging" ]; then + echo "::error::Direct PRs to master are blocked. Use staging branch." + exit 1 + fi + echo "✅ Valid staging → master PR" + else + echo "ℹ️ Not a master PR - branch rules not enforced" + fi + + - name: Validate PR Title + run: | + PR_TITLE="${{ github.event.pull_request.title }}" + PATTERN="^(feat|fix|docs|style|refactor|test|chore|build|ci|perf|revert)(\(.+\))?: .+" + + if [[ ! "$PR_TITLE" =~ $PATTERN ]]; then + echo "::error::PR title must follow: type(scope): description" + echo "Valid formats:" + echo "- feat: add new feature" + echo "- fix(login): resolve auth issue" + echo "Allowed types: feat|fix|docs|style|refactor|test|chore|build|ci|perf|revert" + exit 1 + fi + echo "✅ Valid PR title format" \ No newline at end of file diff --git a/.github/workflows/release-approval.yml b/.github/workflows/release-approval.yml new file mode 100644 index 0000000000..798ee72090 --- /dev/null +++ b/.github/workflows/release-approval.yml @@ -0,0 +1,19 @@ +name: Olake Driver Releaser + +on: + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: "tag to release" + required: true + +jobs: + build_all_drivers: + if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/master') + name: Build and Release Drivers + uses: ./.github/workflows/docker-images.yml + with: + tag: ${{ github.event.inputs.tag || github.event.release.tag_name }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/todo.yml b/.github/workflows/todo.yml new file mode 100644 index 0000000000..e74f5f1990 --- /dev/null +++ b/.github/workflows/todo.yml @@ -0,0 +1,41 @@ +name: TODO Issue CI + +on: + push: + branches: + - "staging" + workflow_dispatch: + inputs: + importAll: + default: false + required: false + type: boolean + description: Enable, if you want to import all TODOs. Runs on checked out branch! Only use if you're sure what you are doing. + +# jobs: +# todos: +# name: Create Issues from TODO +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v3 +# - name: todo-actions +# uses: dtinth/todo-actions@v0.2.0 +# env: +# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# TODO_ACTIONS_MONGO_URL: ${{ secrets.TODO_ACTIONS_MONGO_URL }} + +permissions: + issues: write + repository-projects: read + contents: read + +jobs: + todos: + name: Create Issues from TODO + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: todo-issue + uses: DerJuulsn/todo-issue@v1.1.4 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index c140957d0a..49d38cbfbb 100644 --- a/.gitignore +++ b/.gitignore @@ -80,3 +80,4 @@ resources !dist/src/main/amoro-bin/bin/ !dist/src/main/amoro-bin/conf/ +/docker/kind/data/ \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..821af1ba2f --- /dev/null +++ b/Makefile @@ -0,0 +1,87 @@ +# Makefile for Amoro + Kind (Spark on Kubernetes) +# +# Usage: +# make setup-amoro - One command to start everything (Kind + Docker services + optimizer) +# make stop-amoro - Stop Docker services (Kind cluster persists) +# make teardown - Remove everything (Kind cluster + Docker services + volumes) +# make logs - View Amoro logs +# make status - Show status of cluster and Amoro +# make pods - List Spark pods in kubernetes +# make shell - Open shell in Amoro container + +COMPOSE_DIR := docker/kind +KIND_CLUSTER := amoro-cluster +KUBECTL := kubectl --context kind-$(KIND_CLUSTER) + +.PHONY: setup-amoro stop-amoro teardown logs status pods shell alias help + +# Default target +.DEFAULT_GOAL := help + +help: + @echo "Amoro + Kind (Spark on Kubernetes)" + @echo "" + @echo "Usage:" + @echo " make setup-amoro Start everything (Kind cluster + all services + optimizer)" + @echo " make stop-amoro Stop Docker services (Kind cluster persists)" + @echo " make restart-amoro Restart Docker services" + @echo " make teardown Remove everything (Kind cluster + services + volumes)" + @echo " make logs View Amoro logs" + @echo " make status Show cluster and service status" + @echo " make pods List Spark pods in Kubernetes" + @echo " make shell Shell into Amoro container" + @echo " make alias Set default namespace to spark" + @echo "" + @echo "Access:" + @echo " Amoro Web UI : http://localhost:1630 (admin / admin)" + @echo " MinIO Console: http://localhost:9001 (admin / password)" + @echo "" + +setup-amoro: + @echo "Starting Amoro (Kind cluster + all services)..." + @docker compose -f $(COMPOSE_DIR)/docker-compose.yml up -d + @echo "" + @echo "Exporting Kind kubeconfig to host..." + @kind export kubeconfig --name $(KIND_CLUSTER) 2>/dev/null + @echo "" + @echo "Follow progress: make logs" + @echo "Check status: make status" + +stop-amoro: + @docker compose -f $(COMPOSE_DIR)/docker-compose.yml down + +restart-amoro: + @docker compose -f $(COMPOSE_DIR)/docker-compose.yml down + @docker compose -f $(COMPOSE_DIR)/docker-compose.yml up -d + @kind export kubeconfig --name $(KIND_CLUSTER) 2>/dev/null + +teardown: + @echo "Removing Kind clusters..." + @kind delete cluster --name $(KIND_CLUSTER) 2>/dev/null + @kind delete cluster --name amoro-spark-cluster 2>/dev/null + @echo "Removing Docker services and volumes..." + @docker compose -f $(COMPOSE_DIR)/docker-compose.yml down -v + @echo "Teardown complete." + +logs: + @docker compose -f $(COMPOSE_DIR)/docker-compose.yml logs -f amoro + +status: + @echo "=== Kind Cluster ===" + @kind get clusters 2>/dev/null || echo " No clusters" + @echo "" + @echo "=== Kubernetes Nodes ===" + @$(KUBECTL) get nodes 2>/dev/null || echo " Cannot connect to cluster (run: make setup-amoro)" + @echo "" + @echo "=== Docker Services ===" + @docker compose -f $(COMPOSE_DIR)/docker-compose.yml ps + @echo "" + @echo "=== Spark Pods ===" + @$(KUBECTL) get pods -n spark 2>/dev/null || echo " No pods or cannot connect" + +alias: + @$(KUBECTL) config set-context --current --namespace=spark + @echo "alias k='kubectl'" + +shell: + @docker exec -it amoro /bin/bash diff --git a/docker/kind/config.yaml b/docker/kind/config.yaml new file mode 100644 index 0000000000..74ce85aba4 --- /dev/null +++ b/docker/kind/config.yaml @@ -0,0 +1,175 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ams: + admin-username: admin + admin-password: password + server-bind-host: "0.0.0.0" + # Static IP on iceberg-net so both the local optimizer and K8s Spark pods can + # reach the thrift optimizing service without any manual IP edits. + server-expose-host: "172.30.0.10" + + thrift-server: + max-message-size: 100MB # 104857600 + selector-thread-count: 2 + selector-queue-size: 4 + table-service: + bind-port: 1260 + worker-thread-count: 20 + optimizing-service: + bind-port: 1261 + + http-server: + session-timeout: 7d + bind-port: 1630 + rest-auth-type: token + + refresh-external-catalogs: + interval: 3min # 180000 + thread-count: 10 + queue-size: 1000000 + + refresh-tables: + thread-count: 10 + interval: 1min # 60000 + max-pending-partition-count: 100 # default 100 + + self-optimizing: + commit-thread-count: 10 + runtime-data-keep-days: 30 + runtime-data-expire-interval-hours: 1 + refresh-group-interval: 30s + + optimizer: + heart-beat-timeout: 1min # 60000 + task-ack-timeout: 30s # 30000 + task-execute-timeout: 1h # 3600000 + polling-timeout: 3s # 3000 + max-planning-parallelism: 1 # default 1 + + blocker: + timeout: 1min # 60000 + + # optional features + expire-snapshots: + enabled: true + thread-count: 10 + + clean-orphan-files: + enabled: true + thread-count: 10 + interval: 1d + + clean-dangling-delete-files: + enabled: true + thread-count: 10 + + sync-hive-tables: + enabled: false + thread-count: 10 + + data-expiration: + enabled: true + thread-count: 10 + interval: 1d + + auto-create-tags: + enabled: true + thread-count: 3 + interval: 1min # 60000 + + table-manifest-io: + thread-count: 20 + + catalog-meta-cache: + expiration-interval: 60s + + # Support for encrypted sensitive configuration items + shade: + identifier: default # Built-in support for default/base64. Defaults to "default", indicating no encryption + sensitive-keywords: admin-password;database.password + + overview-cache: + refresh-interval: 3min # 3 min + max-size: 3360 # Keep 7 days history by default, 7 * 24 * 60 / 3 = 3360 + + # PostgreSQL — resolved via Docker Compose DNS (service name "postgres") + database: + type: postgres + jdbc-driver-class: org.postgresql.Driver + url: jdbc:postgresql://postgres:5432/iceberg + username: iceberg + password: password + connection-pool-max-total: 20 + connection-pool-max-idle: 16 + connection-pool-max-wait-millis: 30000 + + terminal: + backend: local + result: + limit: 1000 + stop-on-error: false + session: + timeout: 30min # 1800000 + local: + using-session-catalog-for-hive: false + spark.sql.iceberg.handle-timestamp-without-timezone: false + +# --------------------------------------------------------------------------- +# Optimizer Containers +# --------------------------------------------------------------------------- +# localContainer — runs inside the Amoro JVM; zero external dependencies. +# +# sparkContainer — submits Spark jobs to the Kind K8s cluster. +# The docker-compose "amoro-init" service auto-creates the +# 'spark-container' optimizer group + 1 optimizer using this. +# Requires the apache/amoro-spark-optimizer:latest image to +# be built and loaded into Kind (done automatically by the +# kind-load-image service). +# --------------------------------------------------------------------------- +containers: + - name: localContainer + container-impl: org.apache.amoro.server.manager.LocalOptimizerContainer + properties: + export.JAVA_HOME: "/opt/java/openjdk" # eclipse-temurin JDK path + + - name: sparkContainer + container-impl: org.apache.amoro.server.manager.SparkOptimizerContainer + properties: + spark-home: /opt/spark/ + # Kind control-plane hostname — resolved via Docker DNS (amoro-net) + master: k8s://https://amoro-cluster-control-plane:6443 + deploy-mode: cluster + job-uri: "local:///opt/spark/usrlib/optimizer-job.jar" + export.HADOOP_USER_NAME: spark + export.SPARK_CONF_DIR: /opt/spark/conf/ + spark-conf.spark.kubernetes.container.image: "apache/amoro-spark-optimizer:latest" + spark-conf.spark.kubernetes.container.image.pullPolicy: "IfNotPresent" + spark-conf.spark.kubernetes.namespace: spark + spark-conf.spark.kubernetes.authenticate.driver.serviceAccountName: spark + spark-conf.spark.kubernetes.authenticate.executor.serviceAccountName: spark + spark-conf.spark.dynamicAllocation.enabled: "true" + spark-conf.spark.shuffle.service.enabled: "false" + spark-conf.spark.dynamicAllocation.shuffleTracking.enabled: "true" + spark-conf.spark.driver.memory: "1g" + spark-conf.spark.executor.memory: "1g" + spark-conf.spark.executor.cores: "1" + # AWS/MinIO credentials for S3FileIO (Spark optimizer pods need these) + spark-conf.spark.kubernetes.driver.secretKeyRef.AWS_ACCESS_KEY_ID: "amoro-s3-credentials:access-key-id" + spark-conf.spark.kubernetes.driver.secretKeyRef.AWS_SECRET_ACCESS_KEY: "amoro-s3-credentials:secret-access-key" + spark-conf.spark.kubernetes.executor.secretKeyRef.AWS_ACCESS_KEY_ID: "amoro-s3-credentials:access-key-id" + spark-conf.spark.kubernetes.executor.secretKeyRef.AWS_SECRET_ACCESS_KEY: "amoro-s3-credentials:secret-access-key" \ No newline at end of file diff --git a/docker/kind/docker-compose.yml b/docker/kind/docker-compose.yml new file mode 100644 index 0000000000..02405cdadc --- /dev/null +++ b/docker/kind/docker-compose.yml @@ -0,0 +1,316 @@ + +services: + fusion: + image: olakego/fusion:latest + container_name: olake-fusion + depends_on: + postgres: + condition: service_healthy + kind-setup: + condition: service_completed_successfully + spark-copy: + condition: service_completed_successfully + networks: + iceberg-net: + ipv4_address: 172.30.0.10 + ports: + - "1630:1630" # Web UI + REST API + - "1260:1260" # Thrift table service + - "1261:1261" # Thrift optimizing service + environment: + JVM_XMS: "1024" + JVM_XMX: "2048" + KUBECONFIG: /root/.kube/config + AWS_ACCESS_KEY_ID: admin + AWS_SECRET_ACCESS_KEY: password + volumes: + - ./config.yaml:/usr/local/amoro/conf/config.yaml:ro + - kind-kubeconfig:/root/.kube:ro + - spark-home:/opt/spark:ro + command: ["/entrypoint.sh", "ams"] + tty: true + stdin_open: true + + fusion-init: + image: curlimages/curl + container_name: fusion-init + depends_on: + fusion: + condition: service_started + kind-load-image: + condition: service_completed_successfully + networks: + iceberg-net: + entrypoint: + - /bin/sh + - -c + - | + echo "Waiting for Fusion to be ready..." + until curl -sf http://amoro:1630/ >/dev/null 2>&1; do sleep 2; done + sleep 10 + + echo "Logging in..." + curl -sf -c /tmp/cookies.txt \ + -X POST http://amoro:1630/api/ams/v1/login \ + -H "Content-Type: application/json" \ + -H "X-Request-Source: Web" \ + -d '{"user":"admin","password":"admin"}' + echo "" + + # ---- Optimizer Group (Spark on Kubernetes) ---- + echo "Checking optimizer group..." + GROUPS=$$(curl -s -b /tmp/cookies.txt \ + -H "X-Request-Source: Web" \ + http://amoro:1630/api/ams/v1/optimize/resourceGroups) + + if echo "$$GROUPS" | grep -q '"name":"spark-container"'; then + echo " Optimizer group 'spark-container' already exists — skipping." + else + echo " Creating optimizer group 'spark-container' (Spark on K8s)..." + curl -s -b /tmp/cookies.txt \ + -X POST http://amoro:1630/api/ams/v1/optimize/resourceGroups \ + -H "Content-Type: application/json" \ + -H "X-Request-Source: Web" \ + -d '{"name":"spark-container","container":"sparkContainer","properties":{}}' + echo "" + sleep 5 + fi + + # ---- Optimizer (always ensure at least 1 is running) ---- + echo "Checking optimizers..." + OPTIMIZERS=$$(curl -s -b /tmp/cookies.txt \ + -H "X-Request-Source: Web" \ + "http://amoro:1630/api/ams/v1/optimize/optimizerGroups/spark-container/optimizers?page=1&pageSize=10") + + if echo "$$OPTIMIZERS" | grep -q '"jobStatus":"RUNNING"'; then + echo " Spark optimizer already running — skipping." + else + echo " Scaling out Spark optimizer (parallelism=1)..." + curl -s -b /tmp/cookies.txt \ + -X POST http://amoro:1630/api/ams/v1/optimize/optimizerGroups/spark-container/optimizers \ + -H "Content-Type: application/json" \ + -H "X-Request-Source: Web" \ + -d '{"parallelism":1}' + echo "" + sleep 5 + fi + + # ---- Catalog ---- + echo "Checking catalog..." + CATALOGS=$$(curl -s -b /tmp/cookies.txt \ + -H "X-Request-Source: Web" \ + http://amoro:1630/api/ams/v1/catalogs) + + if echo "$$CATALOGS" | grep -q '"catalogName":"olake_iceberg"'; then + echo " Catalog 'olake_iceberg' already exists — skipping." + else + echo " Creating catalog 'olake_iceberg' (Iceberg JdbcCatalog on Postgres + MinIO)..." + curl -s -b /tmp/cookies.txt \ + -X POST http://amoro:1630/api/ams/v1/catalogs \ + -H "Content-Type: application/json" \ + -H "X-Request-Source: Web" \ + -d '{ + "name":"olake_iceberg", + "type":"custom", + "optimizerGroup":"spark-container", + "tableFormatList":["ICEBERG"], + "storageConfig":{ + "storage.type":"S3", + "storage.s3.endpoint":"http://172.30.0.5:9000", + "storage.s3.region":"us-east-1" + }, + "authConfig":{ + "auth.type":"AKSK", + "auth.aksk.access-key":"admin", + "auth.aksk.secret-key":"password" + }, + "properties":{ + "catalog-impl":"org.apache.iceberg.jdbc.JdbcCatalog", + "uri":"jdbc:postgresql://postgres:5432/iceberg", + "jdbc.user":"iceberg", + "jdbc.password":"password", + "jdbc.driver":"org.postgresql.Driver", + "jdbc.schema-version":"V1", + "warehouse":"s3://warehouse/olake_iceberg/", + "s3.path-style-access":"true", + "io-impl":"org.apache.iceberg.aws.s3.S3FileIO", + "endpoint":"http://172.30.0.5:9000", + "access-key-id":"admin", + "secret-access-key":"password" + }, + "tableProperties":{} + }' + echo "" + fi + + echo "" + echo "============================================" + echo " Fusion is ready!" + echo " Web UI : http://localhost:1630" + echo " Login : admin / password" + echo " Catalog: olake_iceberg" + echo "============================================" + + postgres: + image: postgres:15 + container_name: postgres + networks: + iceberg-net: + ports: + - "5432:5432" + environment: + POSTGRES_USER: iceberg + POSTGRES_PASSWORD: password + POSTGRES_DB: iceberg + healthcheck: + test: ["CMD", "pg_isready", "-U", "iceberg", "-d", "iceberg"] + interval: 3s + timeout: 10s + retries: 10 + start_period: 30s + volumes: + - ./data/postgres_data:/var/lib/postgresql/data + command: postgres -c max_connections=300 + + minio: + image: minio/minio:RELEASE.2025-04-03T14-56-28Z + container_name: minio + networks: + iceberg-net: + ipv4_address: 172.30.0.5 + environment: + MINIO_ROOT_USER: admin + MINIO_ROOT_PASSWORD: password + MINIO_DOMAIN: minio + ports: + - "9000:9000" + - "9001:9001" + volumes: + - ./data/minio_data:/data + command: ["server", "/data", "--console-address", ":9001"] + + mc: + image: minio/mc:RELEASE.2025-04-03T17-07-56Z + container_name: mc + depends_on: + - minio + networks: + iceberg-net: + entrypoint: + - /bin/sh + - -c + - | + until /usr/bin/mc config host add minio http://minio:9000 admin password; do sleep 1; done + /usr/bin/mc mb minio/warehouse --ignore-existing + /usr/bin/mc anonymous set public minio/warehouse + echo "MinIO bucket 'warehouse' is ready." + + kind-setup: + image: docker:27-cli + container_name: fusion-kind-setup + networks: + iceberg-net: + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - kind-kubeconfig:/output + - ./kind-config.yaml:/kind-config.yaml:ro + - ./spark-rbac.yaml:/spark-rbac.yaml:ro + entrypoint: + - /bin/sh + - -c + - | + set -e + + echo "==> Installing kind..." + apk add --no-cache curl >/dev/null 2>&1 + curl -sLo /usr/local/bin/kind https://kind.sigs.k8s.io/dl/v0.23.0/kind-linux-amd64 + chmod +x /usr/local/bin/kind + + echo "==> Creating Kind cluster (fusion-spark-cluster)..." + if kind get clusters 2>/dev/null | grep -q '^fusion-cluster$$'; then + echo " Cluster already exists, skipping creation." + else + kind create cluster --config /kind-config.yaml --wait 120s + fi + + echo "==> Connecting Kind nodes to iceberg-net..." + docker network connect iceberg-net fusion-cluster-control-plane 2>/dev/null || true + for w in $$(docker ps --filter "name=fusion-cluster-worker" --format "{{.Names}}"); do + docker network connect iceberg-net "$$w" 2>/dev/null || true + done + + echo "==> Applying Spark RBAC (namespace + service account)..." + cat /spark-rbac.yaml | docker exec -i fusion-cluster-control-plane \ + kubectl --kubeconfig /etc/kubernetes/admin.conf apply -f - + + echo "==> Generating kubeconfig for Fusion..." + kind get kubeconfig --name fusion-cluster \ + | sed 's|server: .*|server: https://fusion-cluster-control-plane:6443|' \ + > /output/config + chmod 644 /output/config + + echo "==> Kind setup complete!" + + + kind-load-image: + image: docker:27-cli + container_name: fusion-kind-load-image + depends_on: + kind-setup: + condition: service_completed_successfully + volumes: + - /var/run/docker.sock:/var/run/docker.sock + entrypoint: + - /bin/sh + - -c + - | + IMAGE="olakego/fusion-spark:latest" + + if ! docker image inspect $$IMAGE >/dev/null 2>&1; then + echo "==> Image $$IMAGE not found locally — skipping." + exit 0 + fi + + echo "==> Saving $$IMAGE to tarball (once)..." + docker save $$IMAGE -o /tmp/spark-optimizer.tar + echo " Saved ($$(du -h /tmp/spark-optimizer.tar | cut -f1))." + + echo "==> Importing into Kind worker nodes..." + for NODE in $$(docker ps --filter "name=fusion-cluster-worker" --format "{{.Names}}"); do + echo " Importing into $$NODE..." + cat /tmp/spark-optimizer.tar | docker exec -i $$NODE ctr --namespace=k8s.io images import - + done + + rm -f /tmp/spark-optimizer.tar + echo "==> Done — image available on all workers." + + spark-copy: + image: olakego/fusion-spark:latest + container_name: fusion-spark-copy + entrypoint: + - /bin/sh + - -c + - | + if [ -f /spark-vol/.done ]; then + echo "Spark already copied — skipping." + else + echo "Copying Spark to shared volume..." + cp -a /opt/spark/. /spark-vol/ + touch /spark-vol/.done + echo "Spark copy complete." + fi + volumes: + - spark-home:/spark-vol + + +networks: + iceberg-net: + driver: bridge + name: iceberg-net + ipam: + config: + - subnet: 172.30.0.0/24 + +volumes: + kind-kubeconfig: + spark-home: \ No newline at end of file diff --git a/docker/kind/kind-config.yaml b/docker/kind/kind-config.yaml new file mode 100644 index 0000000000..0eead095f3 --- /dev/null +++ b/docker/kind/kind-config.yaml @@ -0,0 +1,40 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: fusion-cluster +networking: + podSubnet: "10.244.0.0/16" + serviceSubnet: "10.96.0.0/12" + disableDefaultCNI: false +nodes: + - role: control-plane + image: kindest/node:v1.28.0 + kubeadmConfigPatches: + - | + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + - | + kind: ClusterConfiguration + apiServer: + certSANs: + - "127.0.0.1" + - "0.0.0.0" + - "localhost" + - "fusion-cluster-control-plane" + - role: worker + image: kindest/node:v1.28.0 + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + max-pods: "110" + - role: worker + image: kindest/node:v1.28.0 + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + max-pods: "110" diff --git a/docker/kind/spark-rbac.yaml b/docker/kind/spark-rbac.yaml new file mode 100644 index 0000000000..d6bd7b8a42 --- /dev/null +++ b/docker/kind/spark-rbac.yaml @@ -0,0 +1,76 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: spark + labels: + name: spark +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: spark + namespace: spark +--- +# AWS/MinIO credentials for Spark optimizer S3 access +apiVersion: v1 +kind: Secret +metadata: + name: fusion-s3-credentials + namespace: spark +type: Opaque +stringData: + access-key-id: admin + secret-access-key: password +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: spark-role +rules: + # Pod management - required for driver/executor pods + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list", "create", "delete", "patch"] + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["pods/status"] + verbs: ["get", "list", "watch"] + # Services - required for driver service discovery + - apiGroups: [""] + resources: ["services"] + verbs: ["get", "list", "watch", "create", "update", "delete"] + # ConfigMaps - required for Spark configuration + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "delete"] + # PVCs - required if using dynamic volume provisioning + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "create", "delete"] + # Secrets - required for pulling images from private registries + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] + # Events - helpful for debugging + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: spark-role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: spark-role +subjects: + - kind: ServiceAccount + name: spark + namespace: spark From 4c014ee53f659601dcd8956d79a8e552fd39181a Mon Sep 17 00:00:00 2001 From: hashcode-ankit Date: Wed, 11 Feb 2026 23:19:18 +0530 Subject: [PATCH 02/46] fix: makefile cluster name --- Makefile | 46 +++++++++++++++++++--------------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index 821af1ba2f..2a1fdb8931 100644 --- a/Makefile +++ b/Makefile @@ -1,44 +1,41 @@ -# Makefile for Amoro + Kind (Spark on Kubernetes) +# Makefile for Fusion + Kind (Spark on Kubernetes) # # Usage: -# make setup-amoro - One command to start everything (Kind + Docker services + optimizer) -# make stop-amoro - Stop Docker services (Kind cluster persists) +# make setup-fusion - One command to start everything (Kind + Docker services + optimizer) +# make stop-fusion - Stop Docker services (Kind cluster persists) # make teardown - Remove everything (Kind cluster + Docker services + volumes) -# make logs - View Amoro logs -# make status - Show status of cluster and Amoro -# make pods - List Spark pods in kubernetes -# make shell - Open shell in Amoro container +# make status - Show status of cluster and Fusion COMPOSE_DIR := docker/kind -KIND_CLUSTER := amoro-cluster +KIND_CLUSTER := fusion-cluster KUBECTL := kubectl --context kind-$(KIND_CLUSTER) -.PHONY: setup-amoro stop-amoro teardown logs status pods shell alias help +.PHONY: setup-fusion stop-fusion teardown logs status pods shell alias help # Default target .DEFAULT_GOAL := help help: - @echo "Amoro + Kind (Spark on Kubernetes)" + @echo "Fusion + Kind (Spark on Kubernetes)" @echo "" @echo "Usage:" - @echo " make setup-amoro Start everything (Kind cluster + all services + optimizer)" - @echo " make stop-amoro Stop Docker services (Kind cluster persists)" - @echo " make restart-amoro Restart Docker services" + @echo " make setup-fusion Start everything (Kind cluster + all services + optimizer)" + @echo " make stop-fusion Stop Docker services (Kind cluster persists)" + @echo " make restart-fusion Restart Docker services" @echo " make teardown Remove everything (Kind cluster + services + volumes)" - @echo " make logs View Amoro logs" + @echo " make logs View Fusion logs" @echo " make status Show cluster and service status" @echo " make pods List Spark pods in Kubernetes" - @echo " make shell Shell into Amoro container" + @echo " make shell Shell into Fusion container" @echo " make alias Set default namespace to spark" @echo "" @echo "Access:" - @echo " Amoro Web UI : http://localhost:1630 (admin / admin)" + @echo " Fusion Web UI : http://localhost:1630 (admin / password)" @echo " MinIO Console: http://localhost:9001 (admin / password)" @echo "" -setup-amoro: - @echo "Starting Amoro (Kind cluster + all services)..." +setup-fusion: + @echo "Starting Fusion (Kind cluster + all services)..." @docker compose -f $(COMPOSE_DIR)/docker-compose.yml up -d @echo "" @echo "Exporting Kind kubeconfig to host..." @@ -47,10 +44,10 @@ setup-amoro: @echo "Follow progress: make logs" @echo "Check status: make status" -stop-amoro: +stop-fusion: @docker compose -f $(COMPOSE_DIR)/docker-compose.yml down -restart-amoro: +restart-fusion: @docker compose -f $(COMPOSE_DIR)/docker-compose.yml down @docker compose -f $(COMPOSE_DIR)/docker-compose.yml up -d @kind export kubeconfig --name $(KIND_CLUSTER) 2>/dev/null @@ -58,20 +55,17 @@ restart-amoro: teardown: @echo "Removing Kind clusters..." @kind delete cluster --name $(KIND_CLUSTER) 2>/dev/null - @kind delete cluster --name amoro-spark-cluster 2>/dev/null + @kind delete cluster --name fusion-spark-cluster 2>/dev/null @echo "Removing Docker services and volumes..." @docker compose -f $(COMPOSE_DIR)/docker-compose.yml down -v @echo "Teardown complete." -logs: - @docker compose -f $(COMPOSE_DIR)/docker-compose.yml logs -f amoro - status: @echo "=== Kind Cluster ===" @kind get clusters 2>/dev/null || echo " No clusters" @echo "" @echo "=== Kubernetes Nodes ===" - @$(KUBECTL) get nodes 2>/dev/null || echo " Cannot connect to cluster (run: make setup-amoro)" + @$(KUBECTL) get nodes 2>/dev/null || echo " Cannot connect to cluster (run: make setup-fusion)" @echo "" @echo "=== Docker Services ===" @docker compose -f $(COMPOSE_DIR)/docker-compose.yml ps @@ -83,5 +77,3 @@ alias: @$(KUBECTL) config set-context --current --namespace=spark @echo "alias k='kubectl'" -shell: - @docker exec -it amoro /bin/bash From 809e73b1b2d8fb88acf4aabdff48f40e5db94db3 Mon Sep 17 00:00:00 2001 From: hashcode-ankit Date: Sat, 14 Feb 2026 20:53:41 +0530 Subject: [PATCH 03/46] chore: testing new release after change in postgres init --- .github/workflows/docker-images.yml | 4 +- .../resources/postgres/ams-postgres-init.sql | 15 ++-- docker/kind/config.yaml | 14 ++-- docker/kind/docker-compose.yml | 71 +++++++++++++++---- 4 files changed, 75 insertions(+), 29 deletions(-) diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index f772a5d4b6..381d0777bc 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -22,6 +22,8 @@ name: Publish Docker Image on: + push: + - "feat/fusion-releaser" workflow_call: inputs: tag: @@ -58,7 +60,7 @@ jobs: - name: Set Docker tags id: meta run: | - VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-latest' }}" + VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-v1' }}" if [ "${{ github.ref }}" = "refs/heads/master" ]; then case "${VERSION_TAG}" in v*) ;; *) echo "::error::On master branch, version tag must start with 'v' (e.g. v1.0.0). Got: ${VERSION_TAG}"; exit 1 ;; esac echo "tags=olakego/fusion:latest,olakego/fusion:${VERSION_TAG}" >> $GITHUB_OUTPUT diff --git a/amoro-ams/src/main/resources/postgres/ams-postgres-init.sql b/amoro-ams/src/main/resources/postgres/ams-postgres-init.sql index 8f13d68625..683356d7a1 100644 --- a/amoro-ams/src/main/resources/postgres/ams-postgres-init.sql +++ b/amoro-ams/src/main/resources/postgres/ams-postgres-init.sql @@ -456,10 +456,11 @@ CREATE TABLE IF NOT EXISTS ha_lease ( CREATE INDEX IF NOT EXISTS idx_ha_lease_expire ON ha_lease (lease_expire_ts); CREATE INDEX IF NOT EXISTS idx_ha_lease_node ON ha_lease (node_id); -COMMENT ON COLUMN service_name IS 'Service name (AMS/TABLE_SERVICE/OPTIMIZING_SERVICE)'; -COMMENT ON COLUMN node_id IS 'Unique node identifier (host:port:uuid)'; -COMMENT ON COLUMN node_ip IS 'Node IP address'; -COMMENT ON COLUMN server_info_json IS 'JSON encoded server info (AmsServerInfo)'; -COMMENT ON COLUMN lease_expire_ts IS 'Lease expiration timestamp (ms since epoch)'; -COMMENT ON COLUMN version IS 'Optimistic lock version of the lease row'; -COMMENT ON COLUMN updated_at IS 'Last update timestamp (ms since epoch)'; \ No newline at end of file +COMMENT ON COLUMN ha_lease.service_name IS 'Service name (AMS/TABLE_SERVICE/OPTIMIZING_SERVICE)'; +COMMENT ON COLUMN ha_lease.node_id IS 'Unique node identifier (host:port:uuid)'; +COMMENT ON COLUMN ha_lease.node_ip IS 'Node IP address'; +COMMENT ON COLUMN ha_lease.server_info_json IS 'JSON encoded server info (AmsServerInfo)'; +COMMENT ON COLUMN ha_lease.lease_expire_ts IS 'Lease expiration timestamp (ms since epoch)'; +COMMENT ON COLUMN ha_lease.version IS 'Optimistic lock version of the lease row'; +COMMENT ON COLUMN ha_lease.updated_at IS 'Last update timestamp (ms since epoch)'; +COMMENT ON TABLE ha_lease IS 'High-availability lease store'; \ No newline at end of file diff --git a/docker/kind/config.yaml b/docker/kind/config.yaml index 74ce85aba4..55bfaa69b6 100644 --- a/docker/kind/config.yaml +++ b/docker/kind/config.yaml @@ -151,13 +151,13 @@ containers: container-impl: org.apache.amoro.server.manager.SparkOptimizerContainer properties: spark-home: /opt/spark/ - # Kind control-plane hostname — resolved via Docker DNS (amoro-net) - master: k8s://https://amoro-cluster-control-plane:6443 + # Kind control-plane hostname — resolved via Docker DNS (iceberg-net) + master: k8s://https://fusion-cluster-control-plane:6443 deploy-mode: cluster job-uri: "local:///opt/spark/usrlib/optimizer-job.jar" export.HADOOP_USER_NAME: spark export.SPARK_CONF_DIR: /opt/spark/conf/ - spark-conf.spark.kubernetes.container.image: "apache/amoro-spark-optimizer:latest" + spark-conf.spark.kubernetes.container.image: "olakego/fusion-spark:latest" spark-conf.spark.kubernetes.container.image.pullPolicy: "IfNotPresent" spark-conf.spark.kubernetes.namespace: spark spark-conf.spark.kubernetes.authenticate.driver.serviceAccountName: spark @@ -169,7 +169,7 @@ containers: spark-conf.spark.executor.memory: "1g" spark-conf.spark.executor.cores: "1" # AWS/MinIO credentials for S3FileIO (Spark optimizer pods need these) - spark-conf.spark.kubernetes.driver.secretKeyRef.AWS_ACCESS_KEY_ID: "amoro-s3-credentials:access-key-id" - spark-conf.spark.kubernetes.driver.secretKeyRef.AWS_SECRET_ACCESS_KEY: "amoro-s3-credentials:secret-access-key" - spark-conf.spark.kubernetes.executor.secretKeyRef.AWS_ACCESS_KEY_ID: "amoro-s3-credentials:access-key-id" - spark-conf.spark.kubernetes.executor.secretKeyRef.AWS_SECRET_ACCESS_KEY: "amoro-s3-credentials:secret-access-key" \ No newline at end of file + spark-conf.spark.kubernetes.driver.secretKeyRef.AWS_ACCESS_KEY_ID: "fusion-s3-credentials:access-key-id" + spark-conf.spark.kubernetes.driver.secretKeyRef.AWS_SECRET_ACCESS_KEY: "fusion-s3-credentials:secret-access-key" + spark-conf.spark.kubernetes.executor.secretKeyRef.AWS_ACCESS_KEY_ID: "fusion-s3-credentials:access-key-id" + spark-conf.spark.kubernetes.executor.secretKeyRef.AWS_SECRET_ACCESS_KEY: "fusion-s3-credentials:secret-access-key" \ No newline at end of file diff --git a/docker/kind/docker-compose.yml b/docker/kind/docker-compose.yml index 02405cdadc..395400896b 100644 --- a/docker/kind/docker-compose.yml +++ b/docker/kind/docker-compose.yml @@ -46,29 +46,29 @@ services: - -c - | echo "Waiting for Fusion to be ready..." - until curl -sf http://amoro:1630/ >/dev/null 2>&1; do sleep 2; done + until curl -sf http://fusion:1630/ >/dev/null 2>&1; do sleep 2; done sleep 10 echo "Logging in..." curl -sf -c /tmp/cookies.txt \ - -X POST http://amoro:1630/api/ams/v1/login \ + -X POST http://fusion:1630/api/ams/v1/login \ -H "Content-Type: application/json" \ -H "X-Request-Source: Web" \ - -d '{"user":"admin","password":"admin"}' + -d '{"user":"admin","password":"password"}' echo "" # ---- Optimizer Group (Spark on Kubernetes) ---- echo "Checking optimizer group..." GROUPS=$$(curl -s -b /tmp/cookies.txt \ -H "X-Request-Source: Web" \ - http://amoro:1630/api/ams/v1/optimize/resourceGroups) + http://fusion:1630/api/ams/v1/optimize/resourceGroups) if echo "$$GROUPS" | grep -q '"name":"spark-container"'; then echo " Optimizer group 'spark-container' already exists — skipping." else echo " Creating optimizer group 'spark-container' (Spark on K8s)..." curl -s -b /tmp/cookies.txt \ - -X POST http://amoro:1630/api/ams/v1/optimize/resourceGroups \ + -X POST http://fusion:1630/api/ams/v1/optimize/resourceGroups \ -H "Content-Type: application/json" \ -H "X-Request-Source: Web" \ -d '{"name":"spark-container","container":"sparkContainer","properties":{}}' @@ -80,14 +80,14 @@ services: echo "Checking optimizers..." OPTIMIZERS=$$(curl -s -b /tmp/cookies.txt \ -H "X-Request-Source: Web" \ - "http://amoro:1630/api/ams/v1/optimize/optimizerGroups/spark-container/optimizers?page=1&pageSize=10") + "http://fusion:1630/api/ams/v1/optimize/optimizerGroups/spark-container/optimizers?page=1&pageSize=10") if echo "$$OPTIMIZERS" | grep -q '"jobStatus":"RUNNING"'; then echo " Spark optimizer already running — skipping." else echo " Scaling out Spark optimizer (parallelism=1)..." curl -s -b /tmp/cookies.txt \ - -X POST http://amoro:1630/api/ams/v1/optimize/optimizerGroups/spark-container/optimizers \ + -X POST http://fusion:1630/api/ams/v1/optimize/optimizerGroups/spark-container/optimizers \ -H "Content-Type: application/json" \ -H "X-Request-Source: Web" \ -d '{"parallelism":1}' @@ -99,14 +99,14 @@ services: echo "Checking catalog..." CATALOGS=$$(curl -s -b /tmp/cookies.txt \ -H "X-Request-Source: Web" \ - http://amoro:1630/api/ams/v1/catalogs) + http://fusion:1630/api/ams/v1/catalogs) if echo "$$CATALOGS" | grep -q '"catalogName":"olake_iceberg"'; then echo " Catalog 'olake_iceberg' already exists — skipping." else echo " Creating catalog 'olake_iceberg' (Iceberg JdbcCatalog on Postgres + MinIO)..." curl -s -b /tmp/cookies.txt \ - -X POST http://amoro:1630/api/ams/v1/catalogs \ + -X POST http://fusion:1630/api/ams/v1/catalogs \ -H "Content-Type: application/json" \ -H "X-Request-Source: Web" \ -d '{ @@ -260,28 +260,71 @@ services: condition: service_completed_successfully volumes: - /var/run/docker.sock:/var/run/docker.sock + tmpfs: + - /tmp:size=3G entrypoint: - /bin/sh - -c - | + set -e IMAGE="olakego/fusion-spark:latest" + IMAGE_REF="docker.io/olakego/fusion-spark:latest" if ! docker image inspect $$IMAGE >/dev/null 2>&1; then echo "==> Image $$IMAGE not found locally — skipping." exit 0 fi - echo "==> Saving $$IMAGE to tarball (once)..." + # ---- Collect worker nodes ---- + NODES=$$(docker ps --filter "name=fusion-cluster-worker" --format "{{.Names}}") + if [ -z "$$NODES" ]; then + echo "==> No worker nodes found — skipping." + exit 0 + fi + + # ---- Per-node check: only import where missing ---- + NEED_IMPORT="" + for NODE in $$NODES; do + if docker exec $$NODE ctr --namespace=k8s.io images ls -q 2>/dev/null | grep -q "$$IMAGE_REF"; then + echo "==> $$NODE: image already present — skipping." + else + NEED_IMPORT="$$NEED_IMPORT $$NODE" + fi + done + + if [ -z "$$NEED_IMPORT" ]; then + echo "==> Image present on all workers — nothing to do." + exit 0 + fi + + # ---- Save image to tmpfs-backed tarball (fast RAM I/O) ---- + echo "==> Saving $$IMAGE to tarball (tmpfs)..." docker save $$IMAGE -o /tmp/spark-optimizer.tar echo " Saved ($$(du -h /tmp/spark-optimizer.tar | cut -f1))." - echo "==> Importing into Kind worker nodes..." - for NODE in $$(docker ps --filter "name=fusion-cluster-worker" --format "{{.Names}}"); do - echo " Importing into $$NODE..." - cat /tmp/spark-optimizer.tar | docker exec -i $$NODE ctr --namespace=k8s.io images import - + # ---- Parallel import into workers that need it ---- + echo "==> Importing into Kind worker nodes (parallel)..." + PIDS="" + for NODE in $$NEED_IMPORT; do + echo " Starting import into $$NODE..." + (cat /tmp/spark-optimizer.tar | docker exec -i $$NODE ctr --namespace=k8s.io images import - && \ + echo " $$NODE: done.") & + PIDS="$$PIDS $$!" + done + + FAIL=0 + for PID in $$PIDS; do + if ! wait $$PID; then + FAIL=1 + fi done rm -f /tmp/spark-optimizer.tar + + if [ $$FAIL -ne 0 ]; then + echo "==> WARNING: Some imports failed!" + exit 1 + fi echo "==> Done — image available on all workers." spark-copy: From 216e3cf35f926a414b5f5af365da77ca5bc9916a Mon Sep 17 00:00:00 2001 From: hashcode-ankit Date: Sat, 14 Feb 2026 20:59:05 +0530 Subject: [PATCH 04/46] chore: branch tag --- .github/workflows/docker-images.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index 381d0777bc..894f9391c8 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -23,7 +23,8 @@ name: Publish Docker Image on: push: - - "feat/fusion-releaser" + branches: + - "feat/fusion-releaser" workflow_call: inputs: tag: From 8b4a6c2be788573a88ec2733eef7423fc2ac592f Mon Sep 17 00:00:00 2001 From: hashcode-ankit Date: Sat, 14 Feb 2026 21:59:17 +0530 Subject: [PATCH 05/46] voila: working perfectly --- .github/workflows/docker-images.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index 894f9391c8..2602f140f7 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -17,14 +17,11 @@ # Modified by Olake by Datazip Inc. in 2026 -# Build and publish Docker images. Triggered by push (dev-latest) or workflow_call with tag (e.g. release). +# Build and publish Docker images. Triggered by push (dev-v1) or workflow_call with tag (e.g. release). name: Publish Docker Image on: - push: - branches: - - "feat/fusion-releaser" workflow_call: inputs: tag: @@ -117,7 +114,7 @@ jobs: - name: Set Docker tags id: meta run: | - VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-latest' }}" + VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-v1' }}" if [ "${{ github.ref }}" = "refs/heads/master" ]; then case "${VERSION_TAG}" in v*) ;; *) echo "::error::On master branch, version tag must start with 'v' (e.g. v1.0.0). Got: ${VERSION_TAG}"; exit 1 ;; esac echo "tags=olakego/fusion-spark:latest,olakego/fusion-spark:${VERSION_TAG}" >> $GITHUB_OUTPUT From c693d2bd0ade0a7e6d395b3ea35ff5bbdcd9b1a1 Mon Sep 17 00:00:00 2001 From: hashcode-ankit Date: Thu, 5 Mar 2026 13:58:56 +0530 Subject: [PATCH 06/46] fix: upgrading version of java scala and spark --- .github/workflows/docker-images.yml | 23 ++++---- CONTRIBUTING.md | 2 +- README.md | 4 +- dev/deps/dependencies-hadoop-3-spark-3.5 | 52 +++++++++---------- docker/amoro/Dockerfile | 4 +- docker/build.sh | 4 +- docker/kind/docker-compose.yml | 24 ++------- docker/optimizer-spark/Dockerfile | 14 ++++- docs/admin-guides/deployment.md | 2 +- pom.xml | 23 +++----- .../content/community/release-guide.md | 2 +- 11 files changed, 72 insertions(+), 82 deletions(-) diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index 2602f140f7..523614aff2 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -17,11 +17,14 @@ # Modified by Olake by Datazip Inc. in 2026 -# Build and publish Docker images. Triggered by push (dev-v1) or workflow_call with tag (e.g. release). +# Build and publish Docker images. Triggered by push (dev-upgrade-v1) or workflow_call with tag (e.g. release). name: Publish Docker Image on: + push: + branches: + - "feat/fusion-releaser" # remove it before merge workflow_call: inputs: tag: @@ -42,10 +45,10 @@ jobs: if: ${{ startsWith(github.repository, 'datazip-inc/') }} steps: - uses: actions/checkout@v3 - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v3 with: - java-version: '11' + java-version: '17' distribution: 'temurin' cache: maven check-latest: false @@ -58,7 +61,7 @@ jobs: - name: Set Docker tags id: meta run: | - VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-v1' }}" + VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-upgrade-v1' }}" if [ "${{ github.ref }}" = "refs/heads/master" ]; then case "${VERSION_TAG}" in v*) ;; *) echo "::error::On master branch, version tag must start with 'v' (e.g. v1.0.0). Got: ${VERSION_TAG}"; exit 1 ;; esac echo "tags=olakego/fusion:latest,olakego/fusion:${VERSION_TAG}" >> $GITHUB_OUTPUT @@ -94,14 +97,14 @@ jobs: if: ${{ startsWith(github.repository, 'datazip-inc/') }} strategy: matrix: - spark: [ "3.5.7" ] # spark version supported - scala: [ "2.12.15" ] # scala version supported + spark: [ "3.5.8" ] # spark version supported + scala: [ "2.13.18" ] # scala version supported steps: - uses: actions/checkout@v3 - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v3 with: - java-version: '11' + java-version: '17' distribution: 'temurin' cache: maven check-latest: false @@ -114,7 +117,7 @@ jobs: - name: Set Docker tags id: meta run: | - VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-v1' }}" + VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-upgrade-v1' }}" if [ "${{ github.ref }}" = "refs/heads/master" ]; then case "${VERSION_TAG}" in v*) ;; *) echo "::error::On master branch, version tag must start with 'v' (e.g. v1.0.0). Got: ${VERSION_TAG}"; exit 1 ;; esac echo "tags=olakego/fusion-spark:latest,olakego/fusion-spark:${VERSION_TAG}" >> $GITHUB_OUTPUT @@ -149,7 +152,7 @@ jobs: && echo "AMORO_VERSION=${AMORO_VERSION}" >> $GITHUB_OUTPUT - name: Build optimizer module with Maven - run: ./mvnw clean package -pl 'amoro-optimizer/amoro-optimizer-spark' -am -e ${SPARK_VERSION} -DskipTests -B -ntp + run: ./mvnw clean package -pl 'amoro-optimizer/amoro-optimizer-spark' -am -e ${SPARK_VERSION} -Dscala.version=${{ matrix.scala }} -Dscala.binary.version=${{ steps.versions.outputs.SCALA_BINARY_VERSION }} -DskipTests -B -ntp - name: Build and Push Spark Optimizer Docker Image to olakego/fusion-spark uses: docker/build-push-action@v4 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6bdbc76522..cdb6d4ace6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,7 +92,7 @@ for reference. The following guide describes how to import the Amoro project into IntelliJ IDEA and deploy it. ### Requirements -+ Java Version: Java 11 is required. ++ Java Version: Java 17 is required. #### Required plugins 1. Go to `Settings` → `Plugins` in IntelliJ IDEA. diff --git a/README.md b/README.md index d4bdf344fd..8806d27a66 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ Amoro contains modules as below: ## Building -Amoro is built using Maven with JDK 8, 11 and 17(required for `amoro-format-mixed/amoro-mixed-trino` module). +Amoro is built using Maven with JDK 17. * Build all modules without `amoro-mixed-trino`: `./mvnw clean package` * Build and skip tests: `./mvnw clean package -DskipTests` @@ -126,7 +126,7 @@ Amoro is built using Maven with JDK 8, 11 and 17(required for `amoro-format-mixe * Build with hadoop 2.x(the default is 3.x) dependencies: `./mvnw clean package -DskipTests -Phadoop2` * Specify Flink version for Flink optimizer(the default is 1.20.0): `./mvnw clean package -DskipTests -Dflink-optimizer.flink-version=1.20.0` * If the version of Flink is below 1.15.0, you also need to add the `-Pflink-optimizer-pre-1.15` parameter: `./mvnw clean package -DskipTests -Pflink-optimizer-pre-1.15 -Dflink-optimizer.flink-version=1.14.6` -* Specify Spark version for Spark optimizer(the default is 3.5.7): `./mvnw clean package -DskipTests -Dspark.version=3.5.7` +* Specify Spark version for Spark optimizer(the default is 3.5.8): `./mvnw clean package -DskipTests -Dspark.version=3.5.8` * Build `amoro-mixed-trino` module under JDK 17: `./mvnw clean package -DskipTests -Pformat-mixed-format-trino,build-mixed-format-trino -pl 'amoro-format-mixed/amoro-mixed-trino' -am`. * Build all modules: `./mvnw clean package -DskipTests -Ptoolchain,build-mixed-format-trino`, besides you need config `toolchains.xml` in `${user.home}/.m2/` dir with content below. * Build a distribution package with all formats integrated: `./mvnw clean package -Psupport-all-formats` diff --git a/dev/deps/dependencies-hadoop-3-spark-3.5 b/dev/deps/dependencies-hadoop-3-spark-3.5 index 2facef79f9..1a2632fbb2 100644 --- a/dev/deps/dependencies-hadoop-3-spark-3.5 +++ b/dev/deps/dependencies-hadoop-3-spark-3.5 @@ -38,7 +38,7 @@ checker-qual/3.42.0//checker-qual-3.42.0.jar checksums-spi/2.24.12//checksums-spi-2.24.12.jar checksums/2.24.12//checksums-2.24.12.jar chill-java/0.10.0//chill-java-0.10.0.jar -chill_2.12/0.10.0//chill_2.12-0.10.0.jar +chill_2.13/0.10.0//chill_2.13-0.10.0.jar commons-beanutils/1.11.0//commons-beanutils-1.11.0.jar commons-cli/1.2//commons-cli-1.2.jar commons-codec/1.17.2//commons-codec-1.17.2.jar @@ -141,8 +141,8 @@ iceberg-data/1.6.1//iceberg-data-1.6.1.jar iceberg-hive-metastore/1.6.1//iceberg-hive-metastore-1.6.1.jar iceberg-orc/1.6.1//iceberg-orc-1.6.1.jar iceberg-parquet/1.6.1//iceberg-parquet-1.6.1.jar -iceberg-spark-3.5_2.12/1.6.1//iceberg-spark-3.5_2.12-1.6.1.jar -iceberg-spark-extensions-3.5_2.12/1.6.1//iceberg-spark-extensions-3.5_2.12-1.6.1.jar +iceberg-spark-3.5_2.13/1.6.1//iceberg-spark-3.5_2.13-1.6.1.jar +iceberg-spark-extensions-3.5_2.13/1.6.1//iceberg-spark-extensions-3.5_2.13-1.6.1.jar icu4j/69.1//icu4j-69.1.jar identity-spi/2.24.12//identity-spi-2.24.12.jar ivy/2.5.1//ivy-2.5.1.jar @@ -152,7 +152,7 @@ jackson-core/2.14.2//jackson-core-2.14.2.jar jackson-databind/2.14.2//jackson-databind-2.14.2.jar jackson-dataformat-yaml/2.14.2//jackson-dataformat-yaml-2.14.2.jar jackson-datatype-jsr310/2.14.2//jackson-datatype-jsr310-2.14.2.jar -jackson-module-scala_2.12/2.14.2//jackson-module-scala_2.12-2.14.2.jar +jackson-module-scala_2.13/2.14.2//jackson-module-scala_2.13-2.14.2.jar jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar @@ -190,10 +190,10 @@ jpam/1.1//jpam-1.1.jar jsch/0.1.55//jsch-0.1.55.jar json-utils/2.24.12//json-utils-2.24.12.jar json/1.8//json-1.8.jar -json4s-ast_2.12/3.7.0-M11//json4s-ast_2.12-3.7.0-M11.jar -json4s-core_2.12/3.7.0-M11//json4s-core_2.12-3.7.0-M11.jar -json4s-jackson_2.12/3.7.0-M11//json4s-jackson_2.12-3.7.0-M11.jar -json4s-scalap_2.12/3.7.0-M11//json4s-scalap_2.12-3.7.0-M11.jar +json4s-ast_2.13/3.7.0-M11//json4s-ast_2.13-3.7.0-M11.jar +json4s-core_2.13/3.7.0-M11//json4s-core_2.13-3.7.0-M11.jar +json4s-jackson_2.13/3.7.0-M11//json4s-jackson_2.13-3.7.0-M11.jar +json4s-scalap_2.13/3.7.0-M11//json4s-scalap_2.13-3.7.0-M11.jar jsqlparser/4.7//jsqlparser-4.7.jar jsr305/3.0.0//jsr305-3.0.0.jar jta/1.1//jta-1.1.jar @@ -339,11 +339,11 @@ regions/2.24.12//regions-2.24.12.jar rocksdbjni/7.10.2//rocksdbjni-7.10.2.jar s3-transfer-manager/2.24.12//s3-transfer-manager-2.24.12.jar s3/2.24.12//s3-2.24.12.jar -scala-collection-compat_2.12/2.7.0//scala-collection-compat_2.12-2.7.0.jar -scala-library/2.12.18//scala-library-2.12.18.jar -scala-parser-combinators_2.12/2.3.0//scala-parser-combinators_2.12-2.3.0.jar -scala-reflect/2.12.18//scala-reflect-2.12.18.jar -scala-xml_2.12/2.1.0//scala-xml_2.12-2.1.0.jar +scala-collection-compat_2.13/2.7.0//scala-collection-compat_2.13-2.7.0.jar +scala-library/2.13.18//scala-library-2.13.18.jar +scala-parser-combinators_2.13/2.3.0//scala-parser-combinators_2.13-2.3.0.jar +scala-reflect/2.13.18//scala-reflect-2.13.18.jar +scala-xml_2.13/2.1.0//scala-xml_2.13-2.1.0.jar sdk-core/2.24.12//sdk-core-2.24.12.jar simpleclient/0.16.0//simpleclient-0.16.0.jar simpleclient_common/0.16.0//simpleclient_common-0.16.0.jar @@ -355,19 +355,19 @@ slf4j-api/2.0.17//slf4j-api-2.0.17.jar snakeyaml-engine/2.10//snakeyaml-engine-2.10.jar snakeyaml/2.2//snakeyaml-2.2.jar snappy-java/1.1.10.5//snappy-java-1.1.10.5.jar -spark-catalyst_2.12/3.5.7//spark-catalyst_2.12-3.5.7.jar -spark-common-utils_2.12/3.5.7//spark-common-utils_2.12-3.5.7.jar -spark-core_2.12/3.5.7//spark-core_2.12-3.5.7.jar -spark-hive_2.12/3.5.7//spark-hive_2.12-3.5.7.jar -spark-kvstore_2.12/3.5.7//spark-kvstore_2.12-3.5.7.jar -spark-launcher_2.12/3.5.7//spark-launcher_2.12-3.5.7.jar -spark-network-common_2.12/3.5.7//spark-network-common_2.12-3.5.7.jar -spark-network-shuffle_2.12/3.5.7//spark-network-shuffle_2.12-3.5.7.jar -spark-sketch_2.12/3.5.7//spark-sketch_2.12-3.5.7.jar -spark-sql-api_2.12/3.5.7//spark-sql-api_2.12-3.5.7.jar -spark-sql_2.12/3.5.7//spark-sql_2.12-3.5.7.jar -spark-tags_2.12/3.5.7//spark-tags_2.12-3.5.7.jar -spark-unsafe_2.12/3.5.7//spark-unsafe_2.12-3.5.7.jar +spark-catalyst_2.13/3.5.8//spark-catalyst_2.13-3.5.8.jar +spark-common-utils_2.13/3.5.8//spark-common-utils_2.13-3.5.8.jar +spark-core_2.13/3.5.8//spark-core_2.13-3.5.8.jar +spark-hive_2.13/3.5.8//spark-hive_2.13-3.5.8.jar +spark-kvstore_2.13/3.5.8//spark-kvstore_2.13-3.5.8.jar +spark-launcher_2.13/3.5.8//spark-launcher_2.13-3.5.8.jar +spark-network-common_2.13/3.5.8//spark-network-common_2.13-3.5.8.jar +spark-network-shuffle_2.13/3.5.8//spark-network-shuffle_2.13-3.5.8.jar +spark-sketch_2.13/3.5.8//spark-sketch_2.13-3.5.8.jar +spark-sql-api_2.13/3.5.8//spark-sql-api_2.13-3.5.8.jar +spark-sql_2.13/3.5.8//spark-sql_2.13-3.5.8.jar +spark-tags_2.13/3.5.8//spark-tags_2.13-3.5.8.jar +spark-unsafe_2.13/3.5.8//spark-unsafe_2.13-3.5.8.jar sqlline/1.3.0//sqlline-1.3.0.jar stream/2.9.6//stream-2.9.6.jar sts/2.24.12//sts-2.24.12.jar diff --git a/docker/amoro/Dockerfile b/docker/amoro/Dockerfile index 8315411c74..7f95832d28 100644 --- a/docker/amoro/Dockerfile +++ b/docker/amoro/Dockerfile @@ -23,7 +23,7 @@ # --tag apache/amoro:tagname # . -FROM eclipse-temurin:11-jdk-jammy AS builder +FROM eclipse-temurin:17-jdk-jammy AS builder # Add the entire project to the build container, unzip it, # and remove flink-optimizer to reduce the container size. @@ -40,7 +40,7 @@ RUN AMORO_VERSION=`cat pom.xml | grep 'amoro-parent' -C 3 | grep -Eo '. && rm -rf /workspace/amoro -FROM eclipse-temurin:11-jdk-jammy +FROM eclipse-temurin:17-jdk-jammy ARG MAVEN_MIRROR=https://repo.maven.apache.org/maven2 diff --git a/docker/build.sh b/docker/build.sh index 3a02e10d51..10a8a0e516 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -27,7 +27,7 @@ cd $CURRENT_DIR AMORO_VERSION=`cat $PROJECT_HOME/pom.xml | grep 'amoro-parent' -C 3 | grep -Eo '.*' | awk -F'[><]' '{print $3}'` FLINK_VERSION=1.20.0 -SPARK_VERSION=3.5.7 +SPARK_VERSION=3.5.8 DEBIAN_MIRROR=http://deb.debian.org APACHE_ARCHIVE=https://archive.apache.org/dist FLINK_OPTIMIZER_JOB_PATH=amoro-optimizer/amoro-optimizer-flink/target/amoro-optimizer-flink-${AMORO_VERSION}-jar-with-dependencies.jar @@ -50,7 +50,7 @@ Images: Options: --flink-version Flink binary release version, default is 1.20.0, format must be x.y.z - --spark-version Spark binary release version, default is 3.5.7, format must be x.y.z + --spark-version Spark binary release version, default is 3.5.8, format must be x.y.z --apache-archive Apache Archive url, default is https://archive.apache.org/dist --debian-mirror Mirror url of debian, default is http://deb.debian.org --maven-mirror Mirror url of maven, default is https://repo.maven.apache.org/maven2 diff --git a/docker/kind/docker-compose.yml b/docker/kind/docker-compose.yml index 395400896b..a679e16724 100644 --- a/docker/kind/docker-compose.yml +++ b/docker/kind/docker-compose.yml @@ -282,30 +282,16 @@ services: exit 0 fi - # ---- Per-node check: only import where missing ---- - NEED_IMPORT="" - for NODE in $$NODES; do - if docker exec $$NODE ctr --namespace=k8s.io images ls -q 2>/dev/null | grep -q "$$IMAGE_REF"; then - echo "==> $$NODE: image already present — skipping." - else - NEED_IMPORT="$$NEED_IMPORT $$NODE" - fi - done - - if [ -z "$$NEED_IMPORT" ]; then - echo "==> Image present on all workers — nothing to do." - exit 0 - fi - - # ---- Save image to tmpfs-backed tarball (fast RAM I/O) ---- + # ---- Always import to refresh mutable tags like ":latest" ---- + # This prevents stale runtime images (e.g. old Java version) from being reused. echo "==> Saving $$IMAGE to tarball (tmpfs)..." docker save $$IMAGE -o /tmp/spark-optimizer.tar echo " Saved ($$(du -h /tmp/spark-optimizer.tar | cut -f1))." - # ---- Parallel import into workers that need it ---- - echo "==> Importing into Kind worker nodes (parallel)..." + # ---- Parallel import into all workers ---- + echo "==> Importing into Kind worker nodes (parallel, force refresh)..." PIDS="" - for NODE in $$NEED_IMPORT; do + for NODE in $$NODES; do echo " Starting import into $$NODE..." (cat /tmp/spark-optimizer.tar | docker exec -i $$NODE ctr --namespace=k8s.io images import - && \ echo " $$NODE: done.") & diff --git a/docker/optimizer-spark/Dockerfile b/docker/optimizer-spark/Dockerfile index 4c06576ddd..1d47d03928 100644 --- a/docker/optimizer-spark/Dockerfile +++ b/docker/optimizer-spark/Dockerfile @@ -14,17 +14,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG SPARK_VERSION=3.5.7 +ARG SPARK_VERSION=3.5.8 FROM apache/spark:${SPARK_VERSION} ARG MAVEN_MIRROR=https://repo.maven.apache.org/maven2 ARG OPTIMIZER_JOB=optimizer-job.jar ARG AWS_VERSION=2.24.12 +ARG TARGETARCH USER root RUN apt-get update \ - && apt-get install -y wget + && apt-get install -y wget openjdk-17-jre-headless \ + && rm -rf /var/lib/apt/lists/* + +# Force Spark optimizer runtime to Java 17 and expose a stable JAVA_HOME path. +RUN rm -rf /opt/java/openjdk \ + && mkdir -p /opt/java \ + && ln -sfn /usr/lib/jvm/java-17-openjdk-${TARGETARCH} /opt/java/openjdk \ + && test -x /opt/java/openjdk/bin/java +ENV JAVA_HOME=/opt/java/openjdk +ENV PATH="${JAVA_HOME}/bin:${PATH}" RUN cd $SPARK_HOME/jars \ && wget ${MAVEN_MIRROR}/software/amazon/awssdk/bundle/${AWS_VERSION}/bundle-${AWS_VERSION}.jar \ diff --git a/docs/admin-guides/deployment.md b/docs/admin-guides/deployment.md index 6a6bb75a6a..488aac7d76 100644 --- a/docs/admin-guides/deployment.md +++ b/docs/admin-guides/deployment.md @@ -30,7 +30,7 @@ You can choose to download the stable release package from [download page](../.. ## System requirements -- Java 11 is required. +- Java 17 is required. - Optional: A RDBMS (PostgreSQL 14.x or higher, MySQL 5.5 or higher) - Optional: ZooKeeper 3.4.x or higher diff --git a/pom.xml b/pom.xml index 2161a1b568..0e613cd348 100644 --- a/pom.xml +++ b/pom.xml @@ -70,8 +70,8 @@ - 11 - 11 + 17 + 17 ${java.source.version} ${java.target.version} 3.9.11 @@ -109,8 +109,8 @@ hadoop-client-api hadoop-client-runtime 2.1.1 - 2.12.15 - 2.12 + 2.13.18 + 2.13 2.33 2.0.17 2.20.0 @@ -131,7 +131,7 @@ 1.9.7 2.24.12 3.10.2 - 3.5.7 + 3.5.8 3.5 4.2.19 2.9.3 @@ -1851,15 +1851,6 @@ - - java11 - - [11,) - - - 11 - - spark-3.3 @@ -1877,7 +1868,7 @@ spark-3.5 - 3.5.7 + 3.5.8 3.5 @@ -1891,7 +1882,7 @@ scala-2.13 - 2.13.8 + 2.13.18 2.13 1.1.1 diff --git a/site/amoro-site/content/community/release-guide.md b/site/amoro-site/content/community/release-guide.md index f936eeb637..cbfca72830 100644 --- a/site/amoro-site/content/community/release-guide.md +++ b/site/amoro-site/content/community/release-guide.md @@ -16,7 +16,7 @@ Please refer to the following link to understand the ASF release process: ### Environmental requirements -- JDK 11 +- JDK 17 - Apache Maven 3.8+ - GnuPG 2.1+ - Git From 40aa4ada5766c517b3cc8565608359c1930a9589 Mon Sep 17 00:00:00 2001 From: hashcode-ankit Date: Thu, 5 Mar 2026 15:31:02 +0530 Subject: [PATCH 07/46] chore: add licence comments --- .github/workflows/docker-images.yml | 9 +++----- .github/workflows/draft-changelog.yml | 18 +++++++++++++++ .../workflows/prevent-direct-master-pr.yml | 18 +++++++++++++++ .github/workflows/release-approval.yml | 18 +++++++++++++++ .github/workflows/todo.yml | 18 +++++++++++++++ Makefile | 22 ++++++++++++++----- .../resources/postgres/ams-postgres-init.sql | 2 ++ docker/amoro/Dockerfile | 3 ++- docker/build.sh | 2 ++ docker/kind/config.yaml | 1 + docker/kind/docker-compose.yml | 18 +++++++++++++++ docker/kind/kind-config.yaml | 18 +++++++++++++++ docker/kind/spark-rbac.yaml | 18 +++++++++++++++ docker/optimizer-spark/Dockerfile | 2 ++ pom.xml | 2 ++ 15 files changed, 156 insertions(+), 13 deletions(-) diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index 523614aff2..a2afc77d1e 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -17,14 +17,11 @@ # Modified by Olake by Datazip Inc. in 2026 -# Build and publish Docker images. Triggered by push (dev-upgrade-v1) or workflow_call with tag (e.g. release). +# Build and publish Docker images. Triggered by push (dev-v1) or workflow_call with tag (e.g. release). name: Publish Docker Image on: - push: - branches: - - "feat/fusion-releaser" # remove it before merge workflow_call: inputs: tag: @@ -61,7 +58,7 @@ jobs: - name: Set Docker tags id: meta run: | - VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-upgrade-v1' }}" + VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-v1' }}" if [ "${{ github.ref }}" = "refs/heads/master" ]; then case "${VERSION_TAG}" in v*) ;; *) echo "::error::On master branch, version tag must start with 'v' (e.g. v1.0.0). Got: ${VERSION_TAG}"; exit 1 ;; esac echo "tags=olakego/fusion:latest,olakego/fusion:${VERSION_TAG}" >> $GITHUB_OUTPUT @@ -117,7 +114,7 @@ jobs: - name: Set Docker tags id: meta run: | - VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-upgrade-v1' }}" + VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-v1' }}" if [ "${{ github.ref }}" = "refs/heads/master" ]; then case "${VERSION_TAG}" in v*) ;; *) echo "::error::On master branch, version tag must start with 'v' (e.g. v1.0.0). Got: ${VERSION_TAG}"; exit 1 ;; esac echo "tags=olakego/fusion-spark:latest,olakego/fusion-spark:${VERSION_TAG}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/draft-changelog.yml b/.github/workflows/draft-changelog.yml index 48a0bc4e87..d5374dd7df 100644 --- a/.github/workflows/draft-changelog.yml +++ b/.github/workflows/draft-changelog.yml @@ -1,3 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modified by Olake by Datazip Inc. in 2026 + name: Draft Releaser From Master on: diff --git a/.github/workflows/prevent-direct-master-pr.yml b/.github/workflows/prevent-direct-master-pr.yml index c34b8ab700..a77e0eb9d9 100644 --- a/.github/workflows/prevent-direct-master-pr.yml +++ b/.github/workflows/prevent-direct-master-pr.yml @@ -1,3 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modified by Olake by Datazip Inc. in 2026 + name: Master Branch Protection on: diff --git a/.github/workflows/release-approval.yml b/.github/workflows/release-approval.yml index 798ee72090..1f5bd4058c 100644 --- a/.github/workflows/release-approval.yml +++ b/.github/workflows/release-approval.yml @@ -1,3 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modified by Olake by Datazip Inc. in 2026 + name: Olake Driver Releaser on: diff --git a/.github/workflows/todo.yml b/.github/workflows/todo.yml index e74f5f1990..faa0c4eaab 100644 --- a/.github/workflows/todo.yml +++ b/.github/workflows/todo.yml @@ -1,3 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modified by Olake by Datazip Inc. in 2026 + name: TODO Issue CI on: diff --git a/Makefile b/Makefile index 2a1fdb8931..dcd1581e11 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,20 @@ -# Makefile for Fusion + Kind (Spark on Kubernetes) # -# Usage: -# make setup-fusion - One command to start everything (Kind + Docker services + optimizer) -# make stop-fusion - Stop Docker services (Kind cluster persists) -# make teardown - Remove everything (Kind cluster + Docker services + volumes) -# make status - Show status of cluster and Fusion +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modified by Olake by Datazip Inc. in 2026 COMPOSE_DIR := docker/kind KIND_CLUSTER := fusion-cluster diff --git a/amoro-ams/src/main/resources/postgres/ams-postgres-init.sql b/amoro-ams/src/main/resources/postgres/ams-postgres-init.sql index 683356d7a1..8ee161ba57 100644 --- a/amoro-ams/src/main/resources/postgres/ams-postgres-init.sql +++ b/amoro-ams/src/main/resources/postgres/ams-postgres-init.sql @@ -12,6 +12,8 @@ -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -- See the License for the specific language governing permissions and -- limitations under the License. +-- +-- Modified by Olake by Datazip Inc. in 2026 CREATE TABLE catalog_metadata ( diff --git a/docker/amoro/Dockerfile b/docker/amoro/Dockerfile index 7f95832d28..99cc49429b 100644 --- a/docker/amoro/Dockerfile +++ b/docker/amoro/Dockerfile @@ -14,7 +14,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +# +# Modified by Olake by Datazip Inc. in 2026 # Usage: # Run the docker command below under project dir. diff --git a/docker/build.sh b/docker/build.sh index 10a8a0e516..42ba498015 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -16,6 +16,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# Modified by Olake by Datazip Inc. in 2026 +# CURRENT_DIR="$( cd "$(dirname "$0")" ; pwd -P )" PROJECT_HOME="$( cd "$CURRENT_DIR/../" ; pwd -P )" diff --git a/docker/kind/config.yaml b/docker/kind/config.yaml index 55bfaa69b6..8bc1ed430d 100644 --- a/docker/kind/config.yaml +++ b/docker/kind/config.yaml @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# Modified by Olake by Datazip Inc. in 2026 ams: admin-username: admin diff --git a/docker/kind/docker-compose.yml b/docker/kind/docker-compose.yml index a679e16724..da6956618d 100644 --- a/docker/kind/docker-compose.yml +++ b/docker/kind/docker-compose.yml @@ -1,4 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modified by Olake by Datazip Inc. in 2026 + services: fusion: image: olakego/fusion:latest diff --git a/docker/kind/kind-config.yaml b/docker/kind/kind-config.yaml index 0eead095f3..28d4acb9d3 100644 --- a/docker/kind/kind-config.yaml +++ b/docker/kind/kind-config.yaml @@ -1,3 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modified by Olake by Datazip Inc. in 2026 + kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 name: fusion-cluster diff --git a/docker/kind/spark-rbac.yaml b/docker/kind/spark-rbac.yaml index d6bd7b8a42..191a3a8962 100644 --- a/docker/kind/spark-rbac.yaml +++ b/docker/kind/spark-rbac.yaml @@ -1,3 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modified by Olake by Datazip Inc. in 2026 + --- apiVersion: v1 kind: Namespace diff --git a/docker/optimizer-spark/Dockerfile b/docker/optimizer-spark/Dockerfile index 1d47d03928..e880c49273 100644 --- a/docker/optimizer-spark/Dockerfile +++ b/docker/optimizer-spark/Dockerfile @@ -13,6 +13,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Modified by Olake by Datazip Inc. in 2026 ARG SPARK_VERSION=3.5.8 diff --git a/pom.xml b/pom.xml index 0e613cd348..e045e565e3 100644 --- a/pom.xml +++ b/pom.xml @@ -15,6 +15,8 @@ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ~ See the License for the specific language governing permissions and ~ limitations under the License. + ~ + ~ Modified by Olake by Datazip Inc. in 2026 --> From 731d88feb8a06d4f36f668c51814fbac20f7d52a Mon Sep 17 00:00:00 2001 From: hashcode-ankit Date: Thu, 5 Mar 2026 15:40:56 +0530 Subject: [PATCH 08/46] chore: changes asked by Badal and some licence updates --- .github/workflows/docker-images.yml | 2 +- .github/workflows/prevent-direct-master-pr.yml | 2 +- .github/workflows/release-approval.yml | 2 +- CONTRIBUTING.md | 2 ++ README.md | 2 ++ 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index a2afc77d1e..b319e5d45a 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -35,7 +35,7 @@ concurrency: cancel-in-progress: true jobs: - docker-amoro: + docker-fusion: name: Push Fusion Docker Image runs-on: ubuntu-latest environment: docker publish diff --git a/.github/workflows/prevent-direct-master-pr.yml b/.github/workflows/prevent-direct-master-pr.yml index a77e0eb9d9..2127b92382 100644 --- a/.github/workflows/prevent-direct-master-pr.yml +++ b/.github/workflows/prevent-direct-master-pr.yml @@ -55,4 +55,4 @@ jobs: echo "Allowed types: feat|fix|docs|style|refactor|test|chore|build|ci|perf|revert" exit 1 fi - echo "✅ Valid PR title format" \ No newline at end of file + echo "✅ Valid PR title format" diff --git a/.github/workflows/release-approval.yml b/.github/workflows/release-approval.yml index 1f5bd4058c..32a0462d52 100644 --- a/.github/workflows/release-approval.yml +++ b/.github/workflows/release-approval.yml @@ -30,7 +30,7 @@ on: jobs: build_all_drivers: if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/master') - name: Build and Release Drivers + name: Build and Release Fusion uses: ./.github/workflows/docker-images.yml with: tag: ${{ github.event.inputs.tag || github.event.release.tag_name }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cdb6d4ace6..8ed31ee744 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,6 +14,8 @@ - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + - + - Modified by Olake by Datazip Inc. in 2026 --> # Contributing diff --git a/README.md b/README.md index 8806d27a66..457dbe17da 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + - + - Modified by Olake by Datazip Inc. in 2026 -->

Amoro logo From 1b2af45eaf3a0a36c56e0b1470a13b50f8160b33 Mon Sep 17 00:00:00 2001 From: hashcode-ankit Date: Thu, 5 Mar 2026 16:08:59 +0530 Subject: [PATCH 09/46] chore: try building from org secrets --- .github/workflows/docker-images.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index b319e5d45a..704aa15d96 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -22,12 +22,23 @@ name: Publish Docker Image on: + push: + branches: + - "feat/fusion-releaser" workflow_call: inputs: tag: description: 'Version tag for the image (e.g. v1.0.0).' type: string required: true + # Use org-level or repo-level secrets; caller must pass with secrets: inherit + secrets: + DOCKER_USERNAME: + description: 'Docker Hub username (org or repo secret)' + required: true + DOCKER_PASSWORD: + description: 'Docker Hub password or token (org or repo secret)' + required: true concurrency: From cafdf1f0a3e4c8f184a0ce440b2744a1204ce81a Mon Sep 17 00:00:00 2001 From: hashcode-ankit Date: Thu, 5 Mar 2026 16:22:35 +0530 Subject: [PATCH 10/46] chore: updating licence text --- .github/workflows/docker-images.yml | 5 +---- .github/workflows/draft-changelog.yml | 2 +- .github/workflows/modification-header-check.yml | 2 +- .github/workflows/prevent-direct-master-pr.yml | 2 +- .github/workflows/release-approval.yml | 2 +- .github/workflows/todo.yml | 2 +- CONTRIBUTING.md | 2 +- Makefile | 2 +- README.md | 2 +- amoro-ams/src/main/resources/postgres/ams-postgres-init.sql | 2 +- docker/amoro/Dockerfile | 2 +- docker/build.sh | 2 +- docker/kind/config.yaml | 2 +- docker/kind/docker-compose.yml | 2 +- docker/kind/kind-config.yaml | 2 +- docker/kind/spark-rbac.yaml | 2 +- docker/optimizer-spark/Dockerfile | 2 +- pom.xml | 2 +- 18 files changed, 18 insertions(+), 21 deletions(-) diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index 704aa15d96..62ed2f06a2 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Modified by Olake by Datazip Inc. in 2026 +# Modified by Datazip Inc. in 2026 # Build and publish Docker images. Triggered by push (dev-v1) or workflow_call with tag (e.g. release). @@ -22,9 +22,6 @@ name: Publish Docker Image on: - push: - branches: - - "feat/fusion-releaser" workflow_call: inputs: tag: diff --git a/.github/workflows/draft-changelog.yml b/.github/workflows/draft-changelog.yml index d5374dd7df..95808e943e 100644 --- a/.github/workflows/draft-changelog.yml +++ b/.github/workflows/draft-changelog.yml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Modified by Olake by Datazip Inc. in 2026 +# Modified by Datazip Inc. in 2026 name: Draft Releaser From Master diff --git a/.github/workflows/modification-header-check.yml b/.github/workflows/modification-header-check.yml index 3ab72fbe49..8de2c03b3c 100644 --- a/.github/workflows/modification-header-check.yml +++ b/.github/workflows/modification-header-check.yml @@ -15,7 +15,7 @@ # limitations under the License. # -# Modified by Datazip Pvt. Ltd. in 2026 +# Modified by Datazip Inc. in 2026 # Original work Copyright The Apache Software Foundation (ASF) name: Modification-Header-Check diff --git a/.github/workflows/prevent-direct-master-pr.yml b/.github/workflows/prevent-direct-master-pr.yml index 2127b92382..948941ae18 100644 --- a/.github/workflows/prevent-direct-master-pr.yml +++ b/.github/workflows/prevent-direct-master-pr.yml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Modified by Olake by Datazip Inc. in 2026 +# Modified by Datazip Inc. in 2026 name: Master Branch Protection diff --git a/.github/workflows/release-approval.yml b/.github/workflows/release-approval.yml index 32a0462d52..cd737c77d6 100644 --- a/.github/workflows/release-approval.yml +++ b/.github/workflows/release-approval.yml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Modified by Olake by Datazip Inc. in 2026 +# Modified by Datazip Inc. in 2026 name: Olake Driver Releaser diff --git a/.github/workflows/todo.yml b/.github/workflows/todo.yml index faa0c4eaab..50cbb1824b 100644 --- a/.github/workflows/todo.yml +++ b/.github/workflows/todo.yml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Modified by Olake by Datazip Inc. in 2026 +# Modified by Datazip Inc. in 2026 name: TODO Issue CI diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8ed31ee744..1c4e019ca8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,7 @@ - See the License for the specific language governing permissions and - limitations under the License. - - - Modified by Olake by Datazip Inc. in 2026 + - Modified by Datazip Inc. in 2026 --> # Contributing diff --git a/Makefile b/Makefile index dcd1581e11..4c104aae06 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Modified by Olake by Datazip Inc. in 2026 +# Modified by Datazip Inc. in 2026 COMPOSE_DIR := docker/kind KIND_CLUSTER := fusion-cluster diff --git a/README.md b/README.md index 457dbe17da..e5b7c2e038 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ - See the License for the specific language governing permissions and - limitations under the License. - - - Modified by Olake by Datazip Inc. in 2026 + - Modified by Datazip Inc. in 2026 -->

Amoro logo diff --git a/amoro-ams/src/main/resources/postgres/ams-postgres-init.sql b/amoro-ams/src/main/resources/postgres/ams-postgres-init.sql index 8ee161ba57..08cd03e01d 100644 --- a/amoro-ams/src/main/resources/postgres/ams-postgres-init.sql +++ b/amoro-ams/src/main/resources/postgres/ams-postgres-init.sql @@ -13,7 +13,7 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- --- Modified by Olake by Datazip Inc. in 2026 +-- Modified by Datazip Inc. in 2026 CREATE TABLE catalog_metadata ( diff --git a/docker/amoro/Dockerfile b/docker/amoro/Dockerfile index 99cc49429b..6ae1611a15 100644 --- a/docker/amoro/Dockerfile +++ b/docker/amoro/Dockerfile @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Modified by Olake by Datazip Inc. in 2026 +# Modified by Datazip Inc. in 2026 # Usage: # Run the docker command below under project dir. diff --git a/docker/build.sh b/docker/build.sh index 42ba498015..5869b057e1 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -16,7 +16,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Modified by Olake by Datazip Inc. in 2026 +# Modified by Datazip Inc. in 2026 # CURRENT_DIR="$( cd "$(dirname "$0")" ; pwd -P )" diff --git a/docker/kind/config.yaml b/docker/kind/config.yaml index 8bc1ed430d..4d8545fdf2 100644 --- a/docker/kind/config.yaml +++ b/docker/kind/config.yaml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Modified by Olake by Datazip Inc. in 2026 +# Modified by Datazip Inc. in 2026 ams: admin-username: admin diff --git a/docker/kind/docker-compose.yml b/docker/kind/docker-compose.yml index da6956618d..ac051b6e5e 100644 --- a/docker/kind/docker-compose.yml +++ b/docker/kind/docker-compose.yml @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Modified by Olake by Datazip Inc. in 2026 +# Modified by Datazip Inc. in 2026 services: fusion: diff --git a/docker/kind/kind-config.yaml b/docker/kind/kind-config.yaml index 28d4acb9d3..24e9649544 100644 --- a/docker/kind/kind-config.yaml +++ b/docker/kind/kind-config.yaml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Modified by Olake by Datazip Inc. in 2026 +# Modified by Datazip Inc. in 2026 kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 diff --git a/docker/kind/spark-rbac.yaml b/docker/kind/spark-rbac.yaml index 191a3a8962..1b6e8bf459 100644 --- a/docker/kind/spark-rbac.yaml +++ b/docker/kind/spark-rbac.yaml @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Modified by Olake by Datazip Inc. in 2026 +# Modified by Datazip Inc. in 2026 --- apiVersion: v1 diff --git a/docker/optimizer-spark/Dockerfile b/docker/optimizer-spark/Dockerfile index e880c49273..2830f6e223 100644 --- a/docker/optimizer-spark/Dockerfile +++ b/docker/optimizer-spark/Dockerfile @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Modified by Olake by Datazip Inc. in 2026 +# Modified by Datazip Inc. in 2026 ARG SPARK_VERSION=3.5.8 diff --git a/pom.xml b/pom.xml index e045e565e3..7c6db61c33 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,7 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. ~ - ~ Modified by Olake by Datazip Inc. in 2026 + ~ Modified by Datazip Inc. in 2026 --> From b4b0ae95ba3b723600614171234f97f630f0088d Mon Sep 17 00:00:00 2001 From: hashcode-ankit Date: Thu, 5 Mar 2026 16:24:29 +0530 Subject: [PATCH 11/46] chore: update regex --- .github/workflows/modification-header-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/modification-header-check.yml b/.github/workflows/modification-header-check.yml index 8de2c03b3c..66c931a5d5 100644 --- a/.github/workflows/modification-header-check.yml +++ b/.github/workflows/modification-header-check.yml @@ -42,7 +42,7 @@ jobs: BASE_BRANCH=${{ github.base_ref }} CHANGED_FILES=$(git diff --name-only origin/$BASE_BRANCH...HEAD) - DATAZIP_MODIFICATION_YEAR_REGEX="Modified by Datazip Pvt\. Ltd\. in [0-9]{4}" + DATAZIP_MODIFICATION_YEAR_REGEX="Modified by Datazip Inc\. in [0-9]{4}" ASF_ORIGINAL_WORK_NOTICE_REGEX="Original work Copyright The Apache Software Foundation \\(ASF\\)" NOT_MODIFIED_FILES="" for file in $CHANGED_FILES; do From 662354a0223c7b332061373f3567e99df659ca08 Mon Sep 17 00:00:00 2001 From: Ankit Sharma <111491139+hash-data@users.noreply.github.com> Date: Thu, 12 Mar 2026 13:53:04 +0530 Subject: [PATCH 12/46] feat: debug mode and major interval configurations (#13) * feat: adding interval for major in fusion * feat: adding debug development mode for amoro * chore: formatting * fix: terminal queries * fix: compaction throwing parquet error * chore: updating header * chore: remove 17 check as well as remove extra conf path * chore: adding plugin folder dir * chore: removing unnecessary makefile things * fix: spotless issues and upgrading parquet as we moved to 17 --- .gitignore | 8 +- .vscode/debug.md | 33 +++++ .vscode/launch.json | 51 ++++++++ .vscode/settings.json | 39 ++++++ Makefile | 115 +++++++++++------- amoro-ams/pom.xml | 23 ++++ .../manager/LocalOptimizerContainer.java | 21 +++- .../server/table/TableConfigurations.java | 7 ++ .../apache/amoro/config/OptimizingConfig.java | 17 +++ .../maintainer/IcebergTableMaintainer.java | 5 +- .../plan/CommonPartitionEvaluator.java | 18 ++- .../apache/amoro/table/TableProperties.java | 6 + .../amoro/spark/test/utils/ScalaTestUtil.java | 6 +- .../v3.5/amoro-mixed-spark-3.5/pom.xml | 3 +- .../spark/MixedFormatSparkExtensions.scala | 17 +-- dist/src/main/amoro-bin/bin/optimizer.sh | 5 +- docker/kind/config.yaml | 6 +- docker/kind/docker-compose.yml | 22 +++- docker/kind/plugins/event-listeners.yaml | 23 ++++ docker/kind/plugins/metric-reporters.yaml | 27 ++++ .../kind/plugins/table-runtime-factories.yaml | 26 ++++ pom.xml | 10 +- 22 files changed, 408 insertions(+), 80 deletions(-) create mode 100644 .vscode/debug.md create mode 100644 .vscode/launch.json create mode 100644 .vscode/settings.json create mode 100644 docker/kind/plugins/event-listeners.yaml create mode 100644 docker/kind/plugins/metric-reporters.yaml create mode 100644 docker/kind/plugins/table-runtime-factories.yaml diff --git a/.gitignore b/.gitignore index 49d38cbfbb..2eb67920f8 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ spark/tmp/ spark/spark-warehouse/ # vscode/eclipse files +.vscode/ .classpath .project bin/ @@ -78,6 +79,7 @@ public resources !javadoc/resources -!dist/src/main/amoro-bin/bin/ -!dist/src/main/amoro-bin/conf/ -/docker/kind/data/ \ No newline at end of file +/dist/src/main/amoro-bin/* +!/dist/src/main/amoro-bin/bin/ +!/dist/src/main/amoro-bin/conf/ +/docker/kind/data \ No newline at end of file diff --git a/.vscode/debug.md b/.vscode/debug.md new file mode 100644 index 0000000000..4b971b98e2 --- /dev/null +++ b/.vscode/debug.md @@ -0,0 +1,33 @@ + + +# Start Fusion In Debug Mode + +## Setup Dist Runtime For Local Optimizer +1. Update Java 17 path for your machine in `.vscode/settings.json` (or set Java 17 in user settings). +2. Run `make setup-debug-mode` (starts local deps, runs `mvn clean install -DskipTests`, extracts dist tar, and syncs only `lib/` to `dist/src/main/amoro-bin/lib`). +3. Start AMS from `launch.json` using `AmoroServiceContainer` (or `AmoroServiceContainer (Optimizer Debug)` when optimizer debug flags are needed). +4. Add/create an optimizer through UI (local container/group). +5. Attach optimizer debugger using `OptimizerStandalone` from `launch.json` (default port `5006`). +6. Add breakpoints and verify they are hit. +7. Try Reloading the project in vs code again if any java issue exist (still not fixed ask for help on slack) + +## Teardown +- Run `make teardown-debug-mode` to stop local deps and clean extracted runtime. diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000000..486e40f500 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modified by Datazip Inc. in 2026 + */ + +{ + "version": "0.2.0", + "configurations": [ + { + "type": "java", + "name": "AmoroServiceContainer", + "request": "launch", + "mainClass": "org.apache.amoro.server.AmoroServiceContainer", + "projectName": "amoro-ams", + "cwd": "${workspaceFolder}", + "console": "integratedTerminal", + "vmArgs": "-Dfile.encoding=UTF-8 -Darrow.memory.allocator=unsafe -XX:+UseZGC -XX:CompileCommand=exclude,io/netty/buffer/PoolChunkList.allocate -XX:CompileCommand=exclude,io/netty/buffer/PoolArena.allocate --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED", + "env": { + "AMORO_HOME": "${workspaceFolder}/dist/src/main/amoro-bin", + "AMORO_CONF_DIR": "${workspaceFolder}/docker/kind", + "CONSOLE_LOG_LEVEL": "info", + "OPTIMIZER_JAVA_OPTS": "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5006", + "AMS_SERVER__EXPOSE__HOST": "127.0.0.1", + "AMS_DATABASE_URL": "jdbc:postgresql://localhost:5432/iceberg" + } + }, + { + "type": "java", + "name": "OptimizerStandalone", + "request": "attach", + "hostName": "localhost", + "port": 5006, + "projectName": "amoro-optimizer-standalone" + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000..4a7ac78947 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modified by Datazip Inc. in 2026 + */ + +{ + "java.compile.nullAnalysis.mode": "automatic", + "java.jdt.ls.java.home": "/Library/Java/JavaVirtualMachines/temurin-17.jdk/Contents/Home", + "java.configuration.runtimes": [ + { + "name": "JavaSE-17", + "path": "/Library/Java/JavaVirtualMachines/temurin-17.jdk/Contents/Home", + "default": true + } + ], + "java.configuration.detectJdksAtStartUp": true, + "java.configuration.updateBuildConfiguration": "automatic", + "java.import.maven.arguments": "-Pskip-dashboard-build -DskipTests -Dspotless.skip=true -Dcheckstyle.skip=true", + "java.import.exclusions": [ + "**/node_modules/**", + "**/.git/**" + ], + "makefile.configureOnOpen": false +} diff --git a/Makefile b/Makefile index 4c104aae06..551855fb82 100644 --- a/Makefile +++ b/Makefile @@ -18,9 +18,11 @@ COMPOSE_DIR := docker/kind KIND_CLUSTER := fusion-cluster -KUBECTL := kubectl --context kind-$(KIND_CLUSTER) +AMORO_DIST_TAR := $(CURDIR)/dist/target/apache-amoro-0.9-SNAPSHOT-bin.tar.gz +AMORO_RUNTIME_HOME := $(CURDIR)/dist/target/amoro-0.9-SNAPSHOT +AMORO_BIN_HOME := $(CURDIR)/dist/src/main/amoro-bin -.PHONY: setup-fusion stop-fusion teardown logs status pods shell alias help +.PHONY: start-fusion-docker clean-fusion-docker start-deps stop-deps prepare-optimizer-lib prepare-debug-runtime setup-debug-mode clean-debug-mode sync-frontend spotless-fix help # Default target .DEFAULT_GOAL := help @@ -29,61 +31,92 @@ help: @echo "Fusion + Kind (Spark on Kubernetes)" @echo "" @echo "Usage:" - @echo " make setup-fusion Start everything (Kind cluster + all services + optimizer)" - @echo " make stop-fusion Stop Docker services (Kind cluster persists)" - @echo " make restart-fusion Restart Docker services" - @echo " make teardown Remove everything (Kind cluster + services + volumes)" - @echo " make logs View Fusion logs" - @echo " make status Show cluster and service status" - @echo " make pods List Spark pods in Kubernetes" - @echo " make shell Shell into Fusion container" - @echo " make alias Set default namespace to spark" + @echo " make start-fusion-docker Start everything (Kind cluster + all services + optimizer) *Before running make sure you have installed KIND*" + @echo " make clean-fusion-docker Remove everything (Kind cluster + services + volumes)" + @echo " make start-deps Start Postgres and Minio for IDE debugging" + @echo " make stop-deps Stop Postgres and Minio" + @echo " make setup-debug-mode deps + build + install to ~/.m2 + lib sync" + @echo " make clean-debug-mode Stop deps + cleanup extracted runtime" + @echo " make sync-frontend Sync built frontend assets to target/ (fixes blank UI without rebuild)" + @echo " make spotless-fix Auto-fix all Spotless (Google Java Format) violations" @echo "" @echo "Access:" @echo " Fusion Web UI : http://localhost:1630 (admin / password)" - @echo " MinIO Console: http://localhost:9001 (admin / password)" + @echo " MinIO Console : http://localhost:9001 (admin / password)" @echo "" -setup-fusion: +start-fusion-docker: @echo "Starting Fusion (Kind cluster + all services)..." - @docker compose -f $(COMPOSE_DIR)/docker-compose.yml up -d + @docker compose -f $(COMPOSE_DIR)/docker-compose.yml --profile prod up -d @echo "" @echo "Exporting Kind kubeconfig to host..." @kind export kubeconfig --name $(KIND_CLUSTER) 2>/dev/null - @echo "" - @echo "Follow progress: make logs" - @echo "Check status: make status" - -stop-fusion: - @docker compose -f $(COMPOSE_DIR)/docker-compose.yml down - -restart-fusion: - @docker compose -f $(COMPOSE_DIR)/docker-compose.yml down - @docker compose -f $(COMPOSE_DIR)/docker-compose.yml up -d - @kind export kubeconfig --name $(KIND_CLUSTER) 2>/dev/null -teardown: +clean-fusion-docker: @echo "Removing Kind clusters..." @kind delete cluster --name $(KIND_CLUSTER) 2>/dev/null @kind delete cluster --name fusion-spark-cluster 2>/dev/null @echo "Removing Docker services and volumes..." - @docker compose -f $(COMPOSE_DIR)/docker-compose.yml down -v + @docker compose -f $(COMPOSE_DIR)/docker-compose.yml --profile prod down -v @echo "Teardown complete." -status: - @echo "=== Kind Cluster ===" - @kind get clusters 2>/dev/null || echo " No clusters" - @echo "" - @echo "=== Kubernetes Nodes ===" - @$(KUBECTL) get nodes 2>/dev/null || echo " Cannot connect to cluster (run: make setup-fusion)" - @echo "" - @echo "=== Docker Services ===" - @docker compose -f $(COMPOSE_DIR)/docker-compose.yml ps +start-deps: + @echo "Starting local dependencies (Postgres & Minio)..." + @docker compose -f docker/kind/docker-compose.yml --profile dev up -d + +stop-deps: + @echo "Stopping local dependencies (Postgres & Minio)..." + @docker compose -f docker/kind/docker-compose.yml --profile dev down + +prepare-debug-runtime: + @echo "Cleaning up stale optimizer logs (prevents RAT license check failure)..." + @rm -rf "$(AMORO_BIN_HOME)/logs/optimizer-local-test-"* + @echo "Removing all target directories to prevent stale/corrupt class files..." + @find "$(CURDIR)" -maxdepth 3 -name target -type d -exec rm -rf {} + 2>/dev/null; true + @echo "Building and installing all modules to local Maven repo (~/.m2)..." + @./mvnw clean install -DskipTests -Drat.skip=true -Dspotless.skip=true -Dcheckstyle.skip=true -B -ntp + @$(MAKE) prepare-optimizer-lib + +prepare-optimizer-lib: + @if [ ! -f "$(AMORO_DIST_TAR)" ]; then \ + echo "Missing distribution tar: $(AMORO_DIST_TAR)"; \ + echo "Build it first with: mvn -DskipTests package"; \ + exit 1; \ + fi + @mkdir -p "$(CURDIR)/dist/target" + @rm -rf "$(AMORO_RUNTIME_HOME)" + @tar -xzf "$(AMORO_DIST_TAR)" -C "$(CURDIR)/dist/target" + @rm -rf "$(AMORO_BIN_HOME)/lib" + @cp -R "$(AMORO_RUNTIME_HOME)/lib" "$(AMORO_BIN_HOME)/lib" + @echo "Synced optimizer libs to: $(AMORO_BIN_HOME)" + +setup-debug-mode: + @echo "Setting up debug mode (deps + build + install to ~/.m2 + lib sync)..." + @$(MAKE) start-deps + @$(MAKE) prepare-debug-runtime @echo "" - @echo "=== Spark Pods ===" - @$(KUBECTL) get pods -n spark 2>/dev/null || echo " No pods or cannot connect" + @echo "Setup complete." + @echo "Next: reload VS Code Java project, then run 'AmoroServiceContainer' from launch.json. Follow .vscode/debug.md" + +clean-debug-mode: + @echo "Tearing down debug mode (deps + extracted runtime cleanup)..." + @$(MAKE) stop-deps + @rm -rf "$(AMORO_RUNTIME_HOME)" + @echo "Teardown complete." -alias: - @$(KUBECTL) config set-context --current --namespace=spark - @echo "alias k='kubectl'" +spotless-fix: + @echo "Running Spotless auto-fix (Google Java Format + import ordering)..." + @./mvnw spotless:apply -B -ntp + @echo "Spotless fix complete." +sync-frontend: + @echo "Syncing frontend assets from src/main/resources/static → target/classes/static ..." + @SRC=amoro-web/src/main/resources/static; \ + DST=amoro-web/target/classes/static; \ + if [ ! -d "$$SRC" ]; then \ + echo "ERROR: $$SRC not found. Run 'pnpm build' inside amoro-web/ first."; \ + exit 1; \ + fi; \ + mkdir -p "$$DST"; \ + cp -r "$$SRC"/. "$$DST"/ + @echo "Done. Refresh http://localhost:1630 in your browser." diff --git a/amoro-ams/pom.xml b/amoro-ams/pom.xml index 9b0f96f4a4..27eb0800dc 100644 --- a/amoro-ams/pom.xml +++ b/amoro-ams/pom.xml @@ -15,6 +15,8 @@ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ~ See the License for the specific language governing permissions and ~ limitations under the License. + ~ + ~ Modified by Datazip Inc. in 2026 --> @@ -355,6 +357,27 @@ runtime + + org.apache.iceberg + iceberg-spark-${spark.major.version}_${scala.binary.version} + ${iceberg.version} + runtime + + + + org.apache.iceberg + iceberg-spark-extensions-${spark.major.version}_${scala.binary.version} + ${iceberg.version} + runtime + + + + org.apache.paimon + paimon-spark-${spark.major.version} + ${paimon.version} + runtime + + org.apache.amoro amoro-optimizer-standalone diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/manager/LocalOptimizerContainer.java b/amoro-ams/src/main/java/org/apache/amoro/server/manager/LocalOptimizerContainer.java index 28d9747db9..93f07b9ae4 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/manager/LocalOptimizerContainer.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/manager/LocalOptimizerContainer.java @@ -14,22 +14,27 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modified by Datazip Inc. in 2026 */ package org.apache.amoro.server.manager; import org.apache.amoro.resource.Resource; import org.apache.amoro.shade.guava32.com.google.common.base.Preconditions; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Map; public class LocalOptimizerContainer extends AbstractOptimizerContainer { private static final Logger LOG = LoggerFactory.getLogger(LocalOptimizerContainer.class); + private static final String ENV_OPTIMIZER_JAVA_OPTS = "OPTIMIZER_JAVA_OPTS"; public static final String JOB_MEMORY_PROPERTY = "memory"; @@ -37,11 +42,23 @@ public class LocalOptimizerContainer extends AbstractOptimizerContainer { protected Map doScaleOut(Resource resource) { String startUpArgs = this.buildOptimizerStartupArgsString(resource); try { - String exportCmd = + String exportLogDirCmd = String.format( " export OPTIMIZER_LOG_DIR_NAME=\"optimizer-%s-%s\" ", resource.getGroupName(), resource.getResourceId()); - String startUpCommand = exportCmd + " && " + startUpArgs; + + List exportCommands = new ArrayList<>(exportSystemProperties()); + String optimizerJavaOpts = System.getenv(ENV_OPTIMIZER_JAVA_OPTS); + if (StringUtils.isNotEmpty(optimizerJavaOpts)) { + exportCommands.add( + String.format("export %s='%s'", ENV_OPTIMIZER_JAVA_OPTS, optimizerJavaOpts)); + } + + String exportCmd = String.join(" && ", exportCommands); + String startUpCommand = + StringUtils.isEmpty(exportCmd) + ? exportLogDirCmd + " && " + startUpArgs + : exportCmd + " && " + exportLogDirCmd + " && " + startUpArgs; String[] cmd = {"/bin/sh", "-c", startUpCommand}; LOG.info("Starting local optimizer using command : {}", startUpCommand); ExecUtil.exec(cmd, new ArrayList<>()); diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/table/TableConfigurations.java b/amoro-ams/src/main/java/org/apache/amoro/server/table/TableConfigurations.java index c891a24f4d..34e015f489 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/table/TableConfigurations.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/table/TableConfigurations.java @@ -14,6 +14,8 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modified by Datazip Inc. in 2026 */ package org.apache.amoro.server.table; @@ -297,6 +299,11 @@ public static OptimizingConfig parseOptimizingConfig(Map propert properties, TableProperties.SELF_OPTIMIZING_MAJOR_TRIGGER_DUPLICATE_RATIO, TableProperties.SELF_OPTIMIZING_MAJOR_TRIGGER_DUPLICATE_RATIO_DEFAULT)) + .setMajorTriggerInterval( + CompatiblePropertyUtil.propertyAsInt( + properties, + TableProperties.SELF_OPTIMIZING_MAJOR_TRIGGER_INTERVAL, + TableProperties.SELF_OPTIMIZING_MAJOR_TRIGGER_INTERVAL_DEFAULT)) .setFullTriggerInterval( CompatiblePropertyUtil.propertyAsInt( properties, diff --git a/amoro-common/src/main/java/org/apache/amoro/config/OptimizingConfig.java b/amoro-common/src/main/java/org/apache/amoro/config/OptimizingConfig.java index 0c743ac6bd..d5b882163a 100644 --- a/amoro-common/src/main/java/org/apache/amoro/config/OptimizingConfig.java +++ b/amoro-common/src/main/java/org/apache/amoro/config/OptimizingConfig.java @@ -14,6 +14,8 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modified by Datazip Inc. in 2026 */ package org.apache.amoro.config; @@ -70,6 +72,9 @@ public class OptimizingConfig { // self-optimizing.major.trigger.duplicate-ratio private double majorDuplicateRatio; + // self-optimizing.major.trigger.interval + private int majorTriggerInterval; + // self-optimizing.full.trigger.interval private int fullTriggerInterval; @@ -242,6 +247,15 @@ public OptimizingConfig setMajorDuplicateRatio(double majorDuplicateRatio) { return this; } + public int getMajorTriggerInterval() { + return majorTriggerInterval; + } + + public OptimizingConfig setMajorTriggerInterval(int majorTriggerInterval) { + this.majorTriggerInterval = majorTriggerInterval; + return this; + } + public int getFullTriggerInterval() { return fullTriggerInterval; } @@ -341,6 +355,7 @@ public boolean equals(Object o) { && minorLeastFileCount == that.minorLeastFileCount && minorLeastInterval == that.minorLeastInterval && Double.compare(that.majorDuplicateRatio, majorDuplicateRatio) == 0 + && majorTriggerInterval == that.majorTriggerInterval && fullTriggerInterval == that.fullTriggerInterval && fullRewriteAllFiles == that.fullRewriteAllFiles && Objects.equal(filter, that.filter) @@ -371,6 +386,7 @@ public int hashCode() { minorLeastFileCount, minorLeastInterval, majorDuplicateRatio, + majorTriggerInterval, fullTriggerInterval, fullRewriteAllFiles, filter, @@ -399,6 +415,7 @@ public String toString() { .add("minorLeastFileCount", minorLeastFileCount) .add("minorLeastInterval", minorLeastInterval) .add("majorDuplicateRatio", majorDuplicateRatio) + .add("majorTriggerInterval", majorTriggerInterval) .add("fullTriggerInterval", fullTriggerInterval) .add("fullRewriteAllFiles", fullRewriteAllFiles) .add("filter", filter) diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/formats/iceberg/maintainer/IcebergTableMaintainer.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/formats/iceberg/maintainer/IcebergTableMaintainer.java index 3d1d4bfb1c..4436c64caf 100644 --- a/amoro-format-iceberg/src/main/java/org/apache/amoro/formats/iceberg/maintainer/IcebergTableMaintainer.java +++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/formats/iceberg/maintainer/IcebergTableMaintainer.java @@ -14,6 +14,8 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modified by Datazip Inc. in 2026 */ package org.apache.amoro.formats.iceberg.maintainer; @@ -686,6 +688,7 @@ public CloseableIterable fileScan( return CloseableIterable.transform( CloseableIterable.withNoopClose(Iterables.concat(dataFiles, deleteFiles)), contentFile -> { + ContentFile file = (ContentFile) contentFile.copyWithoutStats(); Literal literal = getExpireTimestampLiteral( contentFile, @@ -694,7 +697,7 @@ public CloseableIterable fileScan( expirationConfig.getDateTimePattern(), Locale.getDefault()), expirationConfig.getNumberDateFormat(), expireValue); - return new FileEntry(contentFile.copyWithoutStats(), literal); + return new FileEntry(file, literal); }); } diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/plan/CommonPartitionEvaluator.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/plan/CommonPartitionEvaluator.java index 2cd11cbd51..b1718cefb5 100644 --- a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/plan/CommonPartitionEvaluator.java +++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/plan/CommonPartitionEvaluator.java @@ -14,6 +14,8 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modified by Datazip Inc. in 2026 */ package org.apache.amoro.optimizing.plan; @@ -54,6 +56,7 @@ public class CommonPartitionEvaluator implements PartitionEvaluator { protected final long planTime; private final boolean reachFullInterval; + private final boolean reachMajorInterval; // fragment files protected int fragmentFileCount = 0; @@ -112,6 +115,9 @@ public CommonPartitionEvaluator( this.lastMinorOptimizingTime = lastMinorOptimizingTime; this.lastMajorOptimizingTime = lastMajorOptimizingTime; this.lastFullOptimizingTime = lastFullOptimizingTime; + this.reachMajorInterval = + config.getMajorTriggerInterval() >= 0 + && planTime - lastMajorOptimizingTime > config.getMajorTriggerInterval(); this.reachFullInterval = config.getFullTriggerInterval() >= 0 && planTime - lastFullOptimizingTime > config.getFullTriggerInterval(); @@ -401,7 +407,7 @@ public boolean enoughContent() { } public boolean isMajorNecessary() { - return enoughContent() || rewriteSegmentFileCount > 0; + return isMajorIntervalNecessary(); } public boolean isMinorNecessary() { @@ -442,6 +448,16 @@ public boolean anyDeleteExist() { return equalityDeleteFileCount > 0 || posDeleteFileCount > 0; } + private boolean isMajorIntervalNecessary() { + if (!reachMajorInterval) { + return false; + } + + // Interval trigger should still require some pending input to avoid empty major process. + int dataFileCount = fragmentFileCount + getSegmentFileCount(); + return dataFileCount > 1 || anyDeleteExist(); + } + @Override public HealthScoreInfo getHealthScore() { long dataFilesSize = getFragmentFileSize() + getSegmentFileSize(); diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java index 0d804cdec7..76354aee7e 100644 --- a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java +++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java @@ -14,6 +14,8 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modified by Datazip Inc. in 2026 */ package org.apache.amoro.table; @@ -118,6 +120,10 @@ private TableProperties() {} "self-optimizing.major.trigger.duplicate-ratio"; public static final double SELF_OPTIMIZING_MAJOR_TRIGGER_DUPLICATE_RATIO_DEFAULT = 0.1; + public static final String SELF_OPTIMIZING_MAJOR_TRIGGER_INTERVAL = + "self-optimizing.major.trigger.interval"; + public static final int SELF_OPTIMIZING_MAJOR_TRIGGER_INTERVAL_DEFAULT = -1; // not trigger + public static final String SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL = "self-optimizing.full.trigger.interval"; public static final int SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL_DEFAULT = -1; // not trigger diff --git a/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/src/test/java/org/apache/amoro/spark/test/utils/ScalaTestUtil.java b/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/src/test/java/org/apache/amoro/spark/test/utils/ScalaTestUtil.java index 725af2d84d..2a91635b86 100644 --- a/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/src/test/java/org/apache/amoro/spark/test/utils/ScalaTestUtil.java +++ b/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/src/test/java/org/apache/amoro/spark/test/utils/ScalaTestUtil.java @@ -14,18 +14,20 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modified by Datazip Inc. in 2026 */ package org.apache.amoro.spark.test.utils; -import scala.collection.JavaConverters; import scala.collection.Seq; +import scala.jdk.CollectionConverters; import java.util.List; public class ScalaTestUtil { public static Seq seq(List values) { - return JavaConverters.asScalaBuffer(values).toSeq(); + return CollectionConverters.ListHasAsScala(values).asScala(); } } diff --git a/amoro-format-mixed/amoro-mixed-spark/v3.5/amoro-mixed-spark-3.5/pom.xml b/amoro-format-mixed/amoro-mixed-spark/v3.5/amoro-mixed-spark-3.5/pom.xml index e03c59f3cd..00b79e8635 100644 --- a/amoro-format-mixed/amoro-mixed-spark/v3.5/amoro-mixed-spark-3.5/pom.xml +++ b/amoro-format-mixed/amoro-mixed-spark/v3.5/amoro-mixed-spark-3.5/pom.xml @@ -15,6 +15,8 @@ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ~ See the License for the specific language governing permissions and ~ limitations under the License. + ~ + ~ Modified by Datazip Inc. in 2026 --> @@ -272,7 +274,6 @@ org.apache.paimon paimon-spark-${spark.major.version} ${paimon.version} - test org.apache.paimon diff --git a/amoro-format-mixed/amoro-mixed-spark/v3.5/amoro-mixed-spark-3.5/src/main/scala/org/apache/amoro/spark/MixedFormatSparkExtensions.scala b/amoro-format-mixed/amoro-mixed-spark/v3.5/amoro-mixed-spark-3.5/src/main/scala/org/apache/amoro/spark/MixedFormatSparkExtensions.scala index acf956f241..0cbb6055d5 100644 --- a/amoro-format-mixed/amoro-mixed-spark/v3.5/amoro-mixed-spark-3.5/src/main/scala/org/apache/amoro/spark/MixedFormatSparkExtensions.scala +++ b/amoro-format-mixed/amoro-mixed-spark/v3.5/amoro-mixed-spark-3.5/src/main/scala/org/apache/amoro/spark/MixedFormatSparkExtensions.scala @@ -14,14 +14,13 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modified by Datazip Inc. in 2026 */ package org.apache.amoro.spark import org.apache.spark.sql.SparkSessionExtensions -import org.apache.spark.sql.catalyst.analysis.{ProcedureArgumentCoercion, ResolveProcedures} -import org.apache.spark.sql.catalyst.optimizer._ -import org.apache.spark.sql.catalyst.parser.extensions.IcebergSparkSqlExtensionsParser import org.apache.spark.sql.execution.datasources.v2.{ExtendedDataSourceV2Strategy, MixedFormatExtendedDataSourceV2Strategy} import org.apache.amoro.spark.sql.catalyst.analysis._ @@ -57,18 +56,6 @@ class MixedFormatSparkExtensions extends (SparkSessionExtensions => Unit) { // mixed-format strategy rules extensions.injectPlannerStrategy { spark => execution.ExtendedMixedFormatStrategy(spark) } - - // === Iceberg extensions === - - // parser extensions - extensions.injectParser { case (_, parser) => new IcebergSparkSqlExtensionsParser(parser) } - - // analyzer extensions - extensions.injectResolutionRule { spark => ResolveProcedures(spark) } - extensions.injectResolutionRule { _ => ProcedureArgumentCoercion } - - // optimizer extensions - extensions.injectOptimizerRule { _ => ReplaceStaticInvoke } } } diff --git a/dist/src/main/amoro-bin/bin/optimizer.sh b/dist/src/main/amoro-bin/bin/optimizer.sh index 8e01c619c6..905a1f41a8 100755 --- a/dist/src/main/amoro-bin/bin/optimizer.sh +++ b/dist/src/main/amoro-bin/bin/optimizer.sh @@ -16,6 +16,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# Modified by Datazip Inc. in 2026 CURRENT_DIR="$( cd "$(dirname "$0")" ; pwd -P )" @@ -84,8 +85,8 @@ else MODULE_OPTS="" fi -# merge parameter -JAVA_OPTS="$BASE_JVM_OPTS $GC_LOG_OPTS $MODULE_OPTS" +# merge parameters: +JAVA_OPTS="$BASE_JVM_OPTS $GC_LOG_OPTS $MODULE_OPTS $JVM_EXTRA_CONFIG $OPTIMIZER_JAVA_OPTS" RUN_SERVER="org.apache.amoro.optimizer.standalone.StandaloneOptimizer" CMDS="$JAVA_RUN $JAVA_OPTS $RUN_SERVER $ARGS" diff --git a/docker/kind/config.yaml b/docker/kind/config.yaml index 4d8545fdf2..8e27bdb1d7 100644 --- a/docker/kind/config.yaml +++ b/docker/kind/config.yaml @@ -22,6 +22,7 @@ ams: server-bind-host: "0.0.0.0" # Static IP on iceberg-net so both the local optimizer and K8s Spark pods can # reach the thrift optimizing service without any manual IP edits. + # In debug mode this is overridden to 127.0.0.1 via AMS_SERVER__EXPOSE__HOST in launch.json. server-expose-host: "172.30.0.10" thrift-server: @@ -118,6 +119,7 @@ ams: connection-pool-max-total: 20 connection-pool-max-idle: 16 connection-pool-max-wait-millis: 30000 + auto-create-tables: true terminal: backend: local @@ -129,6 +131,7 @@ ams: local: using-session-catalog-for-hive: false spark.sql.iceberg.handle-timestamp-without-timezone: false + spark.sql.codegen.wholeStage: false # --------------------------------------------------------------------------- # Optimizer Containers @@ -145,8 +148,7 @@ ams: containers: - name: localContainer container-impl: org.apache.amoro.server.manager.LocalOptimizerContainer - properties: - export.JAVA_HOME: "/opt/java/openjdk" # eclipse-temurin JDK path + properties: {} - name: sparkContainer container-impl: org.apache.amoro.server.manager.SparkOptimizerContainer diff --git a/docker/kind/docker-compose.yml b/docker/kind/docker-compose.yml index ac051b6e5e..33d551b295 100644 --- a/docker/kind/docker-compose.yml +++ b/docker/kind/docker-compose.yml @@ -1,4 +1,3 @@ - # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -21,6 +20,8 @@ services: fusion: image: olakego/fusion:latest container_name: olake-fusion + profiles: + - prod depends_on: postgres: condition: service_healthy @@ -52,6 +53,8 @@ services: fusion-init: image: curlimages/curl container_name: fusion-init + profiles: + - prod depends_on: fusion: condition: service_started @@ -172,6 +175,9 @@ services: postgres: image: postgres:15 container_name: postgres + profiles: + - prod + - dev networks: iceberg-net: ports: @@ -193,6 +199,9 @@ services: minio: image: minio/minio:RELEASE.2025-04-03T14-56-28Z container_name: minio + profiles: + - prod + - dev networks: iceberg-net: ipv4_address: 172.30.0.5 @@ -210,6 +219,9 @@ services: mc: image: minio/mc:RELEASE.2025-04-03T17-07-56Z container_name: mc + profiles: + - prod + - dev depends_on: - minio networks: @@ -226,6 +238,8 @@ services: kind-setup: image: docker:27-cli container_name: fusion-kind-setup + profiles: + - prod networks: iceberg-net: volumes: @@ -273,6 +287,8 @@ services: kind-load-image: image: docker:27-cli container_name: fusion-kind-load-image + profiles: + - prod depends_on: kind-setup: condition: service_completed_successfully @@ -334,6 +350,8 @@ services: spark-copy: image: olakego/fusion-spark:latest container_name: fusion-spark-copy + profiles: + - prod entrypoint: - /bin/sh - -c @@ -360,4 +378,4 @@ networks: volumes: kind-kubeconfig: - spark-home: \ No newline at end of file + spark-home: diff --git a/docker/kind/plugins/event-listeners.yaml b/docker/kind/plugins/event-listeners.yaml new file mode 100644 index 0000000000..7c705b46ea --- /dev/null +++ b/docker/kind/plugins/event-listeners.yaml @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modified by Datazip Inc. in 2026 + +# This file is the config file for plugin event listener +# configurations of event listeners +event-listeners: + - name: logging-listener + enabled: false diff --git a/docker/kind/plugins/metric-reporters.yaml b/docker/kind/plugins/metric-reporters.yaml new file mode 100644 index 0000000000..2fa5130ff5 --- /dev/null +++ b/docker/kind/plugins/metric-reporters.yaml @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modified by Datazip Inc. in 2026 + +# This file is config file for plugin metric reporter +# configurations of metric reporters + +metric-reporters: +# - name: prometheus-exporter # configs for prometheus exporter +# enabled: false +# properties: +# port: 7001 + diff --git a/docker/kind/plugins/table-runtime-factories.yaml b/docker/kind/plugins/table-runtime-factories.yaml new file mode 100644 index 0000000000..dd03717144 --- /dev/null +++ b/docker/kind/plugins/table-runtime-factories.yaml @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modified by Datazip Inc. in 2026 + +# This file is config file for plugin metric reporter +# configurations of metric reporters + +table-runtime-factories: + - name: default # configs for table runtime factory + enabled: true + priority: 100 + diff --git a/pom.xml b/pom.xml index 7c6db61c33..8dce9bae20 100644 --- a/pom.xml +++ b/pom.xml @@ -127,7 +127,7 @@ 2.2.2 5.7.0 4.11.0 - 1.13.1 + 1.15.2 1.15.2 8.0.33 1.9.7 @@ -398,7 +398,6 @@ ${parquet-avro.version} - org.apache.parquet parquet-jackson @@ -1540,18 +1539,12 @@ ,javax,java,\\# - - tools/maven/copyright.txt - 2.12 tools/maven/scalafmt.conf - - tools/maven/copyright.txt - @@ -1624,6 +1617,7 @@ **/components.d.ts **/Chart.lock dev/deps/** + docker/kind/data/** site/** release/** From ae65e72f43a56ad6baaf99c96efee208375fe77d Mon Sep 17 00:00:00 2001 From: hashcode-ankit Date: Thu, 12 Mar 2026 14:06:29 +0530 Subject: [PATCH 13/46] chore: release dev-v2 and header check --- .github/workflows/docker-images.yml | 8 +++++--- .github/workflows/modification-header-check.yml | 3 +-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index 62ed2f06a2..ea14d3e31c 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -17,12 +17,14 @@ # Modified by Datazip Inc. in 2026 -# Build and publish Docker images. Triggered by push (dev-v1) or workflow_call with tag (e.g. release). +# Build and publish Docker images. Triggered by push (dev-v2) or workflow_call with tag (e.g. release). name: Publish Docker Image on: workflow_call: + branches: + - feat/fusion-releaser inputs: tag: description: 'Version tag for the image (e.g. v1.0.0).' @@ -66,7 +68,7 @@ jobs: - name: Set Docker tags id: meta run: | - VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-v1' }}" + VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-v2' }}" if [ "${{ github.ref }}" = "refs/heads/master" ]; then case "${VERSION_TAG}" in v*) ;; *) echo "::error::On master branch, version tag must start with 'v' (e.g. v1.0.0). Got: ${VERSION_TAG}"; exit 1 ;; esac echo "tags=olakego/fusion:latest,olakego/fusion:${VERSION_TAG}" >> $GITHUB_OUTPUT @@ -122,7 +124,7 @@ jobs: - name: Set Docker tags id: meta run: | - VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-v1' }}" + VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-v2' }}" if [ "${{ github.ref }}" = "refs/heads/master" ]; then case "${VERSION_TAG}" in v*) ;; *) echo "::error::On master branch, version tag must start with 'v' (e.g. v1.0.0). Got: ${VERSION_TAG}"; exit 1 ;; esac echo "tags=olakego/fusion-spark:latest,olakego/fusion-spark:${VERSION_TAG}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/modification-header-check.yml b/.github/workflows/modification-header-check.yml index 66c931a5d5..d6cedfb44c 100644 --- a/.github/workflows/modification-header-check.yml +++ b/.github/workflows/modification-header-check.yml @@ -43,14 +43,13 @@ jobs: BASE_BRANCH=${{ github.base_ref }} CHANGED_FILES=$(git diff --name-only origin/$BASE_BRANCH...HEAD) DATAZIP_MODIFICATION_YEAR_REGEX="Modified by Datazip Inc\. in [0-9]{4}" - ASF_ORIGINAL_WORK_NOTICE_REGEX="Original work Copyright The Apache Software Foundation \\(ASF\\)" NOT_MODIFIED_FILES="" for file in $CHANGED_FILES; do if [[ ! -f "$file" ]]; then continue fi - if ! head -40 "$file" | grep -q -E "$DATAZIP_MODIFICATION_YEAR_REGEX" || ! head -40 "$file" | grep -q -E "$ASF_ORIGINAL_WORK_NOTICE_REGEX"; then + if ! head -40 "$file" | grep -q -E "$DATAZIP_MODIFICATION_YEAR_REGEX"; then NOT_MODIFIED_FILES+="$file"$'\n' fi done From 49e60414feea1c4811f290c715dd447f3ab705aa Mon Sep 17 00:00:00 2001 From: hashcode-ankit Date: Thu, 12 Mar 2026 14:54:16 +0530 Subject: [PATCH 14/46] fix: licence and build image --- .github/workflows/docker-images.yml | 3 ++- .github/workflows/modification-header-check.yml | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index ea14d3e31c..187f711e0d 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -22,9 +22,10 @@ name: Publish Docker Image on: - workflow_call: + push: branches: - feat/fusion-releaser + workflow_call: inputs: tag: description: 'Version tag for the image (e.g. v1.0.0).' diff --git a/.github/workflows/modification-header-check.yml b/.github/workflows/modification-header-check.yml index d6cedfb44c..f7e929f1e0 100644 --- a/.github/workflows/modification-header-check.yml +++ b/.github/workflows/modification-header-check.yml @@ -44,11 +44,16 @@ jobs: CHANGED_FILES=$(git diff --name-only origin/$BASE_BRANCH...HEAD) DATAZIP_MODIFICATION_YEAR_REGEX="Modified by Datazip Inc\. in [0-9]{4}" NOT_MODIFIED_FILES="" + LICENSE_REGEX="Licensed to the Apache Software Foundation|Apache License|SPDX-License-Identifier" for file in $CHANGED_FILES; do if [[ ! -f "$file" ]]; then continue fi + if ! head -40 "$file" | grep -q -E "$LICENSE_REGEX"; then + continue + fi + if ! head -40 "$file" | grep -q -E "$DATAZIP_MODIFICATION_YEAR_REGEX"; then NOT_MODIFIED_FILES+="$file"$'\n' fi From 9a04ea3e25c2747616ee28537350b80531b1c7dc Mon Sep 17 00:00:00 2001 From: hashcode-ankit Date: Thu, 12 Mar 2026 14:56:49 +0530 Subject: [PATCH 15/46] chore: remove image push --- .github/workflows/docker-images.yml | 3 --- docs/admin-guides/deployment.md | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index 187f711e0d..95232a51d2 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -22,9 +22,6 @@ name: Publish Docker Image on: - push: - branches: - - feat/fusion-releaser workflow_call: inputs: tag: diff --git a/docs/admin-guides/deployment.md b/docs/admin-guides/deployment.md index 488aac7d76..b6450ea91c 100644 --- a/docs/admin-guides/deployment.md +++ b/docs/admin-guides/deployment.md @@ -23,6 +23,8 @@ menu: - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + - + - Modified by Datazip Inc. in 2026 --> # Deployment From b1cef9c9903a21bff98ff8943cca3b685290dfe6 Mon Sep 17 00:00:00 2001 From: hashcode-ankit Date: Fri, 13 Mar 2026 15:21:46 +0530 Subject: [PATCH 16/46] chore: build correct images --- .github/workflows/docker-images.yml | 10 +++++++--- docker/optimizer-spark/Dockerfile | 13 ++++++------- pom.xml | 26 ++++++++++++++++++++++++-- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index 95232a51d2..023df3b491 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -17,11 +17,14 @@ # Modified by Datazip Inc. in 2026 -# Build and publish Docker images. Triggered by push (dev-v2) or workflow_call with tag (e.g. release). +# Build and publish Docker images. Triggered by push (dev-v3) or workflow_call with tag (e.g. release). name: Publish Docker Image on: + push: + branches: + - "feat/fusion-releaser" # TODO: remove before merge workflow_call: inputs: tag: @@ -66,7 +69,7 @@ jobs: - name: Set Docker tags id: meta run: | - VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-v2' }}" + VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-v3' }}" if [ "${{ github.ref }}" = "refs/heads/master" ]; then case "${VERSION_TAG}" in v*) ;; *) echo "::error::On master branch, version tag must start with 'v' (e.g. v1.0.0). Got: ${VERSION_TAG}"; exit 1 ;; esac echo "tags=olakego/fusion:latest,olakego/fusion:${VERSION_TAG}" >> $GITHUB_OUTPUT @@ -122,7 +125,7 @@ jobs: - name: Set Docker tags id: meta run: | - VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-v2' }}" + VERSION_TAG="${{ github.event_name == 'workflow_call' && inputs.tag || 'dev-v3' }}" if [ "${{ github.ref }}" = "refs/heads/master" ]; then case "${VERSION_TAG}" in v*) ;; *) echo "::error::On master branch, version tag must start with 'v' (e.g. v1.0.0). Got: ${VERSION_TAG}"; exit 1 ;; esac echo "tags=olakego/fusion-spark:latest,olakego/fusion-spark:${VERSION_TAG}" >> $GITHUB_OUTPUT @@ -173,6 +176,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} build-args: | SPARK_VERSION=${{ matrix.spark }} + SPARK_JAVA_TAG=${{ matrix.spark }}-java17 OPTIMIZER_JOB=amoro-optimizer/amoro-optimizer-spark/target/amoro-optimizer-spark-${{ env.SPARK_MAJOR_VERSION}}_${{ env.SCALA_BINARY_VERSION}}-${{ env.AMORO_VERSION }}-jar-with-dependencies.jar diff --git a/docker/optimizer-spark/Dockerfile b/docker/optimizer-spark/Dockerfile index 2830f6e223..5365dc48d9 100644 --- a/docker/optimizer-spark/Dockerfile +++ b/docker/optimizer-spark/Dockerfile @@ -17,8 +17,12 @@ # Modified by Datazip Inc. in 2026 ARG SPARK_VERSION=3.5.8 +# Use the explicit java17 tag to guarantee Java 17 runtime. +# apache/spark:3.5.8-java17 and apache/spark:3.5.8 share the same digest today, +# but the plain short tag can resolve to a cached Java 11 layer on Kubernetes nodes. +ARG SPARK_JAVA_TAG=${SPARK_VERSION}-java17 -FROM apache/spark:${SPARK_VERSION} +FROM apache/spark:${SPARK_JAVA_TAG} ARG MAVEN_MIRROR=https://repo.maven.apache.org/maven2 ARG OPTIMIZER_JOB=optimizer-job.jar @@ -27,14 +31,9 @@ ARG TARGETARCH USER root RUN apt-get update \ - && apt-get install -y wget openjdk-17-jre-headless \ + && apt-get install -y --no-install-recommends wget \ && rm -rf /var/lib/apt/lists/* -# Force Spark optimizer runtime to Java 17 and expose a stable JAVA_HOME path. -RUN rm -rf /opt/java/openjdk \ - && mkdir -p /opt/java \ - && ln -sfn /usr/lib/jvm/java-17-openjdk-${TARGETARCH} /opt/java/openjdk \ - && test -x /opt/java/openjdk/bin/java ENV JAVA_HOME=/opt/java/openjdk ENV PATH="${JAVA_HOME}/bin:${PATH}" diff --git a/pom.xml b/pom.xml index 8dce9bae20..befd1f1277 100644 --- a/pom.xml +++ b/pom.xml @@ -105,7 +105,8 @@ 3.3.1 1.6.1 - 1.2.0 + + 1.1.1 3.1.3 3.4.2 hadoop-client-api @@ -527,6 +528,26 @@ + + + org.apache.hadoop + hadoop-auth + ${hadoop.version} + + + org.slf4j + * + + + org.apache.logging.log4j + * + + + + commons-beanutils commons-beanutils @@ -1542,7 +1563,7 @@ - 2.12 + 2.13 tools/maven/scalafmt.conf @@ -1873,6 +1894,7 @@ 2.12.15 2.12 + 1.2.0 From 2980bb89af8d53395e1bb8d704e8648210c5ac5b Mon Sep 17 00:00:00 2001 From: hashcode-ankit Date: Fri, 13 Mar 2026 15:22:51 +0530 Subject: [PATCH 17/46] chore: removing build tag, prev commit fixed all terminal and optimizer related issues --- .github/workflows/docker-images.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index 023df3b491..b6218022b7 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -22,9 +22,6 @@ name: Publish Docker Image on: - push: - branches: - - "feat/fusion-releaser" # TODO: remove before merge workflow_call: inputs: tag: From b742973882a51504b4b755cb90da4ecb1e6c1963 Mon Sep 17 00:00:00 2001 From: badalprasadsingh Date: Fri, 13 Mar 2026 16:45:39 +0530 Subject: [PATCH 18/46] feat: logs API Signed-off-by: badalprasadsingh --- .../server/dashboard/DashboardServer.java | 11 ++ .../dashboard/controller/LogController.java | 104 ++++++++++++++ .../amoro/optimizer/common/DriverLogger.java | 115 ++++++++++++++++ .../amoro/optimizer/common/TaskLogger.java | 129 ++++++++++++++++++ .../spark/SparkOptimizerExecutor.java | 50 ++++++- .../spark/SparkOptimizingTaskFunction.java | 21 ++- docker/amoro/entrypoint.sh | 5 +- docker/kind/config.yaml | 10 +- docker/kind/docker-compose.yml | 2 + docker/kind/kind-config.yaml | 9 ++ 10 files changed, 446 insertions(+), 10 deletions(-) create mode 100644 amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/LogController.java create mode 100644 amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/DriverLogger.java create mode 100644 amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/TaskLogger.java diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java index e465e65e76..b35650cd4d 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java @@ -43,6 +43,7 @@ import org.apache.amoro.server.dashboard.controller.ApiTokenController; import org.apache.amoro.server.dashboard.controller.CatalogController; import org.apache.amoro.server.dashboard.controller.HealthCheckController; +import org.apache.amoro.server.dashboard.controller.LogController; import org.apache.amoro.server.dashboard.controller.LoginController; import org.apache.amoro.server.dashboard.controller.OptimizerController; import org.apache.amoro.server.dashboard.controller.OptimizerGroupController; @@ -90,6 +91,7 @@ public class DashboardServer { private final VersionController versionController; private final OverviewController overviewController; private final ApiTokenController apiTokenController; + private final LogController logController; private final PasswdAuthenticationProvider basicAuthProvider; private final TokenAuthenticationProvider jwtAuthProvider; @@ -120,6 +122,7 @@ public DashboardServer( this.overviewController = new OverviewController(manager); APITokenManager apiTokenManager = new APITokenManager(); this.apiTokenController = new ApiTokenController(apiTokenManager); + this.logController = new LogController(); String authType = serviceConfig.get(AmoroManagementConf.HTTP_SERVER_REST_AUTH_TYPE); this.basicAuthProvider = @@ -396,6 +399,14 @@ private EndpointGroup apiGroup() { post("/calculate/signature", apiTokenController::calculateSignature); post("/calculate/encryptString", apiTokenController::getEncryptStringFromQueryParam); }); + + // log apis + path( + "/logs", + () -> { + get("/driver/{processId}", logController::getDriverLog); + get("/tasks/{processId}/{taskId}", logController::getTaskLog); + }); }; } diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/LogController.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/LogController.java new file mode 100644 index 0000000000..1c6abdb903 --- /dev/null +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/LogController.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.server.dashboard.controller; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; + +import org.apache.amoro.server.dashboard.response.OkResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.javalin.http.Context; + +public class LogController { + private static final Logger LOG = LoggerFactory.getLogger(LogController.class); + private static final String LOG_BASE_DIR = "/usr/local/amoro/logs/compaction"; + + public void getDriverLog(Context ctx) { + String processId = ctx.pathParam("processId"); + + Path logFilePath = Paths.get(LOG_BASE_DIR, processId, "driver.log"); + + Map response = new HashMap<>(); + response.put("processId", processId); + response.put("logType", "driver"); + response.put("logFilePath", logFilePath.toString()); + + if (!Files.exists(logFilePath)) { + response.put("exists", false); + response.put("content", ""); + response.put("message", "Driver log file not found"); + ctx.json(OkResponse.of(response)); + return; + } + + try { + String content = Files.readString(logFilePath); + response.put("exists", true); + response.put("content", content); + response.put("size", Files.size(logFilePath)); + ctx.json(OkResponse.of(response)); + } catch (IOException e) { + LOG.error("Failed to read driver log file: {}", logFilePath, e); + response.put("exists", true); + response.put("content", ""); + response.put("error", "Failed to read driver log file: " + e.getMessage()); + ctx.json(OkResponse.of(response)); + } + } + + public void getTaskLog(Context ctx) { + String processId = ctx.pathParam("processId"); + String taskId = ctx.pathParam("taskId"); + + Path logFilePath = Paths.get(LOG_BASE_DIR, processId, taskId + ".log"); + + Map response = new HashMap<>(); + response.put("processId", processId); + response.put("taskId", taskId); + response.put("logFilePath", logFilePath.toString()); + + if (!Files.exists(logFilePath)) { + response.put("exists", false); + response.put("content", ""); + response.put("message", "Log file not found"); + ctx.json(OkResponse.of(response)); + return; + } + + try { + String content = Files.readString(logFilePath); + response.put("exists", true); + response.put("content", content); + response.put("size", Files.size(logFilePath)); + ctx.json(OkResponse.of(response)); + } catch (IOException e) { + LOG.error("Failed to read log file: {}", logFilePath, e); + response.put("exists", true); + response.put("content", ""); + response.put("error", "Failed to read log file: " + e.getMessage()); + ctx.json(OkResponse.of(response)); + } + } +} diff --git a/amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/DriverLogger.java b/amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/DriverLogger.java new file mode 100644 index 0000000000..3870258aa8 --- /dev/null +++ b/amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/DriverLogger.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.optimizer.common; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +/** Logger that writes driver-level logs to a single file per process ID. */ +public class DriverLogger implements AutoCloseable { + private static final DateTimeFormatter TIMESTAMP_FORMAT = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"); + private static final String LOG_BASE_DIR = "/mnt/amoro-logs/compaction"; + + private final long processId; + private final String tableName; + private transient PrintWriter writer; + private transient Path logFile; + + public DriverLogger(long processId, String tableName) { + this.processId = processId; + this.tableName = tableName; + } + + private synchronized void ensureWriter() throws IOException { + if (writer == null) { + Path processDir = Paths.get(LOG_BASE_DIR, String.valueOf(processId)); + Files.createDirectories(processDir); + + logFile = processDir.resolve("driver.log"); + File file = logFile.toFile(); + writer = new PrintWriter(new BufferedWriter(new FileWriter(file, true)), true); + } + } + + public void info(String message, Object... args) { + log("INFO", String.format(message, args), null); + } + + public void warn(String message, Object... args) { + log("WARN", String.format(message, args), null); + } + + public void error(String message, Throwable t) { + log("ERROR", message, t); + } + + public void error(String message, Object... args) { + log("ERROR", String.format(message, args), null); + } + + private synchronized void log(String level, String message, Throwable throwable) { + try { + ensureWriter(); + String timestamp = LocalDateTime.now().format(TIMESTAMP_FORMAT); + String logLine = + String.format( + "%s %-5s [DRIVER|P:%d|Table:%s] - %s", + timestamp, level, processId, tableName != null ? tableName : "unknown", message); + writer.println(logLine); + + if (throwable != null) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + throwable.printStackTrace(pw); + writer.println(sw.toString()); + } + writer.flush(); + } catch (IOException e) { + System.err.println("Failed to write to driver log: " + e.getMessage()); + e.printStackTrace(); + } + } + + @Override + public synchronized void close() { + if (writer != null) { + writer.flush(); + writer.close(); + writer = null; + } + } + + public long getProcessId() { + return processId; + } + + public String getTableName() { + return tableName; + } +} diff --git a/amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/TaskLogger.java b/amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/TaskLogger.java new file mode 100644 index 0000000000..40ee3254fb --- /dev/null +++ b/amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/TaskLogger.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.optimizer.common; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.Serializable; +import java.io.StringWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +// writes task-specific logs to separate files organized by process ID +public class TaskLogger implements AutoCloseable, Serializable { + private static final long serialVersionUID = 1L; + private static final DateTimeFormatter TIMESTAMP_FORMAT = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"); + private static final String LOG_BASE_DIR = "/mnt/amoro-logs/compaction"; + + private final long processId; + private final int taskId; + private final String tableName; + private transient PrintWriter writer; + private transient Path logFile; + + public TaskLogger(long processId, int taskId, String tableName) { + this.processId = processId; + this.taskId = taskId; + // why do we need tableName, can we please remove it + this.tableName = tableName; + } + + private synchronized void ensureWriter() throws IOException { + if (writer == null) { + Path processDir = Paths.get(LOG_BASE_DIR, String.valueOf(processId)); + Files.createDirectories(processDir); + + logFile = processDir.resolve(taskId + ".log"); + File file = logFile.toFile(); + writer = new PrintWriter(new BufferedWriter(new FileWriter(file, true)), true); + } + } + + public void info(String message, Object... args) { + log("INFO", String.format(message, args), null); + } + + public void warn(String message, Object... args) { + log("WARN", String.format(message, args), null); + } + + public void error(String message, Throwable t) { + log("ERROR", message, t); + } + + public void error(String message, Object... args) { + log("ERROR", String.format(message, args), null); + } + + private synchronized void log(String level, String message, Throwable throwable) { + try { + ensureWriter(); + String timestamp = LocalDateTime.now().format(TIMESTAMP_FORMAT); + String logLine = + String.format( + "%s %-5s [P:%d|T:%d|Table:%s] - %s", + timestamp, + level, + processId, + taskId, + tableName != null ? tableName : "unknown", + message); + writer.println(logLine); + + if (throwable != null) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + throwable.printStackTrace(pw); + writer.println(sw.toString()); + } + writer.flush(); + } catch (IOException e) { + System.err.println("Failed to write to task log: " + e.getMessage()); + e.printStackTrace(); + } + } + + @Override + public synchronized void close() { + if (writer != null) { + writer.flush(); + writer.close(); + writer = null; + } + } + + public long getProcessId() { + return processId; + } + + public int getTaskId() { + return taskId; + } + + public String getTableName() { + return tableName; + } +} diff --git a/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java b/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java index 91eb9a9e56..7904d859bb 100644 --- a/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java +++ b/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java @@ -18,10 +18,14 @@ package org.apache.amoro.optimizer.spark; +import java.util.List; + import org.apache.amoro.api.OptimizingTask; import org.apache.amoro.api.OptimizingTaskResult; +import org.apache.amoro.optimizer.common.DriverLogger; import org.apache.amoro.optimizer.common.OptimizerConfig; import org.apache.amoro.optimizer.common.OptimizerExecutor; +import org.apache.amoro.optimizer.common.TaskLogger; import org.apache.amoro.optimizing.RewriteFilesInput; import org.apache.amoro.optimizing.TableOptimizing; import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; @@ -31,8 +35,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; - /** * The {@code SparkOptimizerExecutor} takes OptimizingTask from AMS and wraps it as a spark job, * then submit to the spark environment. @@ -53,25 +55,48 @@ protected OptimizingTaskResult executeTask(OptimizingTask task) { OptimizingTaskResult result; String threadName = Thread.currentThread().getName(); long startTime = System.currentTimeMillis(); - try { + + long processId = task.getTaskId().getProcessId(); + int taskId = task.getTaskId().getTaskId(); + String tableName = extractTableName(task); + + try (DriverLogger driverLogger = new DriverLogger(processId, tableName); + TaskLogger taskLogger = new TaskLogger(processId, taskId, tableName)) { + driverLogger.info("Starting compaction process for table: %s", tableName); + driverLogger.info("Process ID: %d, Task ID: %d, Thread: %s", processId, taskId, threadName); + + taskLogger.info("Starting task execution for table: %s", tableName); + taskLogger.info("Task ID: %s, Thread: %s", task.getTaskId(), threadName); ImmutableList of = ImmutableList.of(task); jsc.setJobDescription(jobDescription(task)); SparkOptimizingTaskFunction taskFunction = - new SparkOptimizingTaskFunction(getConfig(), threadId); + new SparkOptimizingTaskFunction(getConfig(), threadId, taskLogger); List results = jsc.parallelize(of, 1).map(taskFunction).collect(); result = results.get(0); + + long duration = System.currentTimeMillis() - startTime; + taskLogger.info("Task completed successfully in %d ms", duration); + driverLogger.info("Task %d completed successfully in %d ms", taskId, duration); LOG.info( "Optimizer executor[{}] executed task[{}] and cost {} ms", threadName, task.getTaskId(), - System.currentTimeMillis() - startTime); + duration); return result; } catch (Throwable r) { + long duration = System.currentTimeMillis() - startTime; + try (DriverLogger errorDriverLogger = new DriverLogger(processId, tableName); + TaskLogger errorLogger = new TaskLogger(processId, taskId, tableName)) { + errorDriverLogger.error("Process failed after %d ms", duration); + errorDriverLogger.error("Error details:", r); + errorLogger.error("Task execution failed after %d ms", duration); + errorLogger.error("Error details:", r); + } LOG.error( "Optimizer executor[{}] executed task[{}] failed, and cost {} ms", threadName, task.getTaskId(), - (System.currentTimeMillis() - startTime), + duration, r); result = new OptimizingTaskResult(task.getTaskId(), threadId); result.setErrorMessage(ExceptionUtil.getErrorMessage(r, ERROR_MESSAGE_MAX_LENGTH)); @@ -93,4 +118,17 @@ private String jobDescription(OptimizingTask task) { } return description; } + + private String extractTableName(OptimizingTask task) { + try { + TableOptimizing.OptimizingInput input = + SerializationUtil.simpleDeserialize(task.getTaskInput()); + if (input instanceof RewriteFilesInput) { + return ((RewriteFilesInput) input).getTable().name(); + } + } catch (Exception e) { + LOG.warn("Failed to extract table name from task", e); + } + return "unknown"; + } } diff --git a/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizingTaskFunction.java b/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizingTaskFunction.java index 5bb88648f7..1b251ea5a8 100644 --- a/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizingTaskFunction.java +++ b/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizingTaskFunction.java @@ -22,6 +22,7 @@ import org.apache.amoro.api.OptimizingTaskResult; import org.apache.amoro.optimizer.common.OptimizerConfig; import org.apache.amoro.optimizer.common.OptimizerExecutor; +import org.apache.amoro.optimizer.common.TaskLogger; import org.apache.spark.api.java.function.Function; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,14 +35,30 @@ public class SparkOptimizingTaskFunction implements Function Date: Fri, 13 Mar 2026 17:33:39 +0530 Subject: [PATCH 19/46] fix: minor Signed-off-by: badalprasadsingh --- .../server/dashboard/controller/LogController.java | 11 +++++------ .../apache/amoro/optimizer/common/DriverLogger.java | 4 +--- .../org/apache/amoro/optimizer/common/TaskLogger.java | 9 +-------- .../amoro/optimizer/spark/SparkOptimizerExecutor.java | 8 ++++---- 4 files changed, 11 insertions(+), 21 deletions(-) diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/LogController.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/LogController.java index 1c6abdb903..99c36392dd 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/LogController.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/LogController.java @@ -18,6 +18,11 @@ package org.apache.amoro.server.dashboard.controller; +import io.javalin.http.Context; +import org.apache.amoro.server.dashboard.response.OkResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -25,12 +30,6 @@ import java.util.HashMap; import java.util.Map; -import org.apache.amoro.server.dashboard.response.OkResponse; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.javalin.http.Context; - public class LogController { private static final Logger LOG = LoggerFactory.getLogger(LogController.class); private static final String LOG_BASE_DIR = "/usr/local/amoro/logs/compaction"; diff --git a/amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/DriverLogger.java b/amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/DriverLogger.java index 3870258aa8..0c538b2116 100644 --- a/amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/DriverLogger.java +++ b/amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/DriverLogger.java @@ -78,9 +78,7 @@ private synchronized void log(String level, String message, Throwable throwable) ensureWriter(); String timestamp = LocalDateTime.now().format(TIMESTAMP_FORMAT); String logLine = - String.format( - "%s %-5s [DRIVER|P:%d|Table:%s] - %s", - timestamp, level, processId, tableName != null ? tableName : "unknown", message); + String.format("%s %-5s [DRIVER|P:%d] - %s", timestamp, level, processId, message); writer.println(logLine); if (throwable != null) { diff --git a/amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/TaskLogger.java b/amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/TaskLogger.java index 40ee3254fb..bf942441e0 100644 --- a/amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/TaskLogger.java +++ b/amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/TaskLogger.java @@ -83,14 +83,7 @@ private synchronized void log(String level, String message, Throwable throwable) ensureWriter(); String timestamp = LocalDateTime.now().format(TIMESTAMP_FORMAT); String logLine = - String.format( - "%s %-5s [P:%d|T:%d|Table:%s] - %s", - timestamp, - level, - processId, - taskId, - tableName != null ? tableName : "unknown", - message); + String.format("%s %-5s [P:%d|T:%d] - %s", timestamp, level, processId, taskId, message); writer.println(logLine); if (throwable != null) { diff --git a/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java b/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java index 7904d859bb..1f79476c72 100644 --- a/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java +++ b/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java @@ -18,8 +18,6 @@ package org.apache.amoro.optimizer.spark; -import java.util.List; - import org.apache.amoro.api.OptimizingTask; import org.apache.amoro.api.OptimizingTaskResult; import org.apache.amoro.optimizer.common.DriverLogger; @@ -35,6 +33,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; + /** * The {@code SparkOptimizerExecutor} takes OptimizingTask from AMS and wraps it as a spark job, * then submit to the spark environment. @@ -62,10 +62,10 @@ protected OptimizingTaskResult executeTask(OptimizingTask task) { try (DriverLogger driverLogger = new DriverLogger(processId, tableName); TaskLogger taskLogger = new TaskLogger(processId, taskId, tableName)) { - driverLogger.info("Starting compaction process for table: %s", tableName); + driverLogger.info("Starting compaction process"); driverLogger.info("Process ID: %d, Task ID: %d, Thread: %s", processId, taskId, threadName); - taskLogger.info("Starting task execution for table: %s", tableName); + taskLogger.info("Starting task execution"); taskLogger.info("Task ID: %s, Thread: %s", task.getTaskId(), threadName); ImmutableList of = ImmutableList.of(task); jsc.setJobDescription(jobDescription(task)); From 6e630a0c1c2eebe57f84fb4656869862c4ef55ff Mon Sep 17 00:00:00 2001 From: badalprasadsingh Date: Tue, 17 Mar 2026 09:34:56 +0530 Subject: [PATCH 20/46] use optimizer-spark/log4j2 for routing into desired log files Signed-off-by: badalprasadsingh --- .../server/dashboard/DashboardServer.java | 14 +- .../dashboard/controller/LogController.java | 105 +++++++------- amoro-common/pom.xml | 10 ++ .../amoro/log/OptimizingTaskLogContext.java | 47 +++++++ .../AbstractRewriteFilesExecutor.java | 53 ++++--- .../amoro/optimizer/common/DriverLogger.java | 113 --------------- .../amoro/optimizer/common/TaskLogger.java | 122 ---------------- .../spark/SparkOptimizerExecutor.java | 50 +++---- .../spark/SparkOptimizingTaskFunction.java | 39 ++++-- .../conf/optimize/log4j2-routing.xml | 132 ++++++++++++++++++ docker/amoro/entrypoint.sh | 6 +- docker/kind/config.yaml | 6 +- docker/kind/docker-compose.yml | 1 + docker/optimizer-spark/Dockerfile | 22 ++- docker/optimizer-spark/log4j2.xml | 97 +++++++++++++ 15 files changed, 454 insertions(+), 363 deletions(-) create mode 100644 amoro-common/src/main/java/org/apache/amoro/log/OptimizingTaskLogContext.java delete mode 100644 amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/DriverLogger.java delete mode 100644 amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/TaskLogger.java create mode 100644 dist/src/main/amoro-bin/conf/optimize/log4j2-routing.xml create mode 100644 docker/optimizer-spark/log4j2.xml diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java index b35650cd4d..ce02bd3f5d 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java @@ -141,6 +141,7 @@ public DashboardServer( } private volatile String indexHtml = null; + // read index.html content public String getIndexFileContent() { if (indexHtml == null) { @@ -183,13 +184,14 @@ public Consumer configStaticFiles() { // staticFiles.headers = Map.of(...); // headers that will be set for the files staticFiles.skipFileFunction = req -> false; - // you can use this to skip certain files in the dir, based on the HttpServletRequest + // you can use this to skip certain files in the dir, based on the + // HttpServletRequest }; } public EndpointGroup endpoints() { return () -> { - /*backend routers*/ + /* backend routers */ path( "", () -> { @@ -400,12 +402,11 @@ private EndpointGroup apiGroup() { post("/calculate/encryptString", apiTokenController::getEncryptStringFromQueryParam); }); - // log apis + // logs api path( "/logs", () -> { - get("/driver/{processId}", logController::getDriverLog); - get("/tasks/{processId}/{taskId}", logController::getTaskLog); + get("/process/{processId}", logController::getProcessLogs); }); }; } @@ -446,7 +447,7 @@ public void preHandleRequest(Context ctx) { public void handleException(Exception e, Context ctx) { if (e instanceof ForbiddenException) { - // request doesn't start with /ams is page request. we return index.html + // request doesn't start with /ams is page request. we return index.html if (!ctx.req.getRequestURI().startsWith("/api/ams")) { ctx.html(getIndexFileContent()); } else { @@ -482,6 +483,7 @@ public void handleException(Exception e, Context ctx) { "/swagger-docs", "/api/ams/v1/api/token/calculate/signature", "/api/ams/v1/api/token/calculate/encryptString", + "/api/ams/v1/logs/*", RestCatalogService.ICEBERG_REST_API_PREFIX + "/*" }; diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/LogController.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/LogController.java index 99c36392dd..6d6572d156 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/LogController.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/LogController.java @@ -24,80 +24,85 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; public class LogController { private static final Logger LOG = LoggerFactory.getLogger(LogController.class); - private static final String LOG_BASE_DIR = "/usr/local/amoro/logs/compaction"; + private static final String LOG_BASE_DIR = "/mnt/amoro-logs/compaction"; - public void getDriverLog(Context ctx) { + public void getProcessLogs(Context ctx) { String processId = ctx.pathParam("processId"); - - Path logFilePath = Paths.get(LOG_BASE_DIR, processId, "driver.log"); + Path processDir = Paths.get(LOG_BASE_DIR, processId); Map response = new HashMap<>(); response.put("processId", processId); - response.put("logType", "driver"); - response.put("logFilePath", logFilePath.toString()); - if (!Files.exists(logFilePath)) { + if (!Files.exists(processDir) || !Files.isDirectory(processDir)) { response.put("exists", false); - response.put("content", ""); - response.put("message", "Driver log file not found"); + response.put("message", "Process log directory not found"); + response.put("driverLog", null); + response.put("taskLogs", new ArrayList<>()); ctx.json(OkResponse.of(response)); return; } - try { - String content = Files.readString(logFilePath); - response.put("exists", true); - response.put("content", content); - response.put("size", Files.size(logFilePath)); - ctx.json(OkResponse.of(response)); - } catch (IOException e) { - LOG.error("Failed to read driver log file: {}", logFilePath, e); - response.put("exists", true); - response.put("content", ""); - response.put("error", "Failed to read driver log file: " + e.getMessage()); - ctx.json(OkResponse.of(response)); - } - } - - public void getTaskLog(Context ctx) { - String processId = ctx.pathParam("processId"); - String taskId = ctx.pathParam("taskId"); + response.put("exists", true); - Path logFilePath = Paths.get(LOG_BASE_DIR, processId, taskId + ".log"); + // Read driver log + Path driverLogPath = processDir.resolve("driver.log"); + Map driverLog = new HashMap<>(); + if (Files.exists(driverLogPath)) { + try { + driverLog.put("exists", true); + driverLog.put("content", Files.readString(driverLogPath)); + driverLog.put("size", Files.size(driverLogPath)); + } catch (IOException e) { + LOG.error("Failed to read driver log: {}", driverLogPath, e); + driverLog.put("exists", true); + driverLog.put("error", "Failed to read: " + e.getMessage()); + } + } else { + driverLog.put("exists", false); + } + response.put("driverLog", driverLog); - Map response = new HashMap<>(); - response.put("processId", processId); - response.put("taskId", taskId); - response.put("logFilePath", logFilePath.toString()); + // Read all task logs + List> taskLogs = new ArrayList<>(); + try (DirectoryStream stream = Files.newDirectoryStream(processDir, "*.log")) { + for (Path taskLogPath : stream) { + String fileName = taskLogPath.getFileName().toString(); + if (fileName.equals("driver.log")) { + continue; // Skip driver log + } - if (!Files.exists(logFilePath)) { - response.put("exists", false); - response.put("content", ""); - response.put("message", "Log file not found"); - ctx.json(OkResponse.of(response)); - return; - } + String taskId = fileName.replace(".log", ""); + Map taskLog = new HashMap<>(); + taskLog.put("taskId", taskId); - try { - String content = Files.readString(logFilePath); - response.put("exists", true); - response.put("content", content); - response.put("size", Files.size(logFilePath)); - ctx.json(OkResponse.of(response)); + try { + taskLog.put("exists", true); + taskLog.put("content", Files.readString(taskLogPath)); + taskLog.put("size", Files.size(taskLogPath)); + taskLogs.add(taskLog); + } catch (IOException e) { + LOG.error("Failed to read task log: {}", taskLogPath, e); + taskLog.put("exists", true); + taskLog.put("error", "Failed to read: " + e.getMessage()); + taskLogs.add(taskLog); + } + } } catch (IOException e) { - LOG.error("Failed to read log file: {}", logFilePath, e); - response.put("exists", true); - response.put("content", ""); - response.put("error", "Failed to read log file: " + e.getMessage()); - ctx.json(OkResponse.of(response)); + LOG.error("Failed to list task logs in directory: {}", processDir, e); } + + response.put("taskLogs", taskLogs); + ctx.json(OkResponse.of(response)); } } diff --git a/amoro-common/pom.xml b/amoro-common/pom.xml index 2b21ff9997..49f0b25198 100644 --- a/amoro-common/pom.xml +++ b/amoro-common/pom.xml @@ -145,6 +145,16 @@ commons-lang3 + + org.apache.logging.log4j + log4j-api + + + + org.apache.logging.log4j + log4j-core + + org.apache.curator diff --git a/amoro-common/src/main/java/org/apache/amoro/log/OptimizingTaskLogContext.java b/amoro-common/src/main/java/org/apache/amoro/log/OptimizingTaskLogContext.java new file mode 100644 index 0000000000..5a58949855 --- /dev/null +++ b/amoro-common/src/main/java/org/apache/amoro/log/OptimizingTaskLogContext.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.amoro.log; + +import org.apache.logging.log4j.ThreadContext; + +public class OptimizingTaskLogContext { + + public static final String PROCESS_ID_KEY = "processId"; + public static final String TASK_ID_KEY = "taskId"; + public static final String LOG_FILE_PATH_KEY = "logFilePath"; + + private static final ThreadLocal CONTEXT_SET = ThreadLocal.withInitial(() -> false); + + public static void setContext(long processId, int taskId) { + CONTEXT_SET.set(true); + ThreadContext.put(PROCESS_ID_KEY, String.valueOf(processId)); + ThreadContext.put(TASK_ID_KEY, String.valueOf(taskId)); + } + + public static void clearContext() { + CONTEXT_SET.remove(); + ThreadContext.remove(PROCESS_ID_KEY); + ThreadContext.remove(TASK_ID_KEY); + ThreadContext.remove(LOG_FILE_PATH_KEY); + } + + public static boolean isContextSet() { + return CONTEXT_SET.get(); + } +} diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/AbstractRewriteFilesExecutor.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/AbstractRewriteFilesExecutor.java index 4446619f92..1b3ba3d960 100644 --- a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/AbstractRewriteFilesExecutor.java +++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/AbstractRewriteFilesExecutor.java @@ -25,6 +25,7 @@ import org.apache.amoro.data.DataTreeNode; import org.apache.amoro.io.AuthenticatedFileIO; import org.apache.amoro.io.writer.SetTreeNode; +import org.apache.amoro.log.OptimizingTaskLogContext; import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; import org.apache.amoro.table.MixedTable; import org.apache.amoro.table.TableProperties; @@ -96,28 +97,48 @@ public AbstractRewriteFilesExecutor( @Override public RewriteFilesOutput execute() { - LOG.info("Start processing table optimize task: {}", input); - - List dataFiles = new ArrayList<>(); - List deleteFiles = new ArrayList<>(); + boolean shouldClearContext = false; + if (!OptimizingTaskLogContext.isContextSet() + && properties.containsKey(TaskProperties.PROCESS_ID)) { + try { + long processId = Long.parseLong(properties.get(TaskProperties.PROCESS_ID)); + int taskId = + properties.containsKey("taskId") ? Integer.parseInt(properties.get("taskId")) : -1; + OptimizingTaskLogContext.setContext(processId, taskId); + shouldClearContext = true; + } catch (Exception e) { + LOG.warn("Failed to set logging context in AbstractRewriteFilesExecutor", e); + } + } - long startTime = System.currentTimeMillis(); try { - if (!ArrayUtils.isEmpty(input.rePosDeletedDataFiles())) { - deleteFiles = io.doAs(this::equalityToPosition); - } + LOG.info("Start processing table optimize task: {}", input); + + List dataFiles = new ArrayList<>(); + List deleteFiles = new ArrayList<>(); - if (!ArrayUtils.isEmpty(input.rewrittenDataFiles())) { - dataFiles = io.doAs(this::rewriterDataFiles); + long startTime = System.currentTimeMillis(); + try { + if (!ArrayUtils.isEmpty(input.rePosDeletedDataFiles())) { + deleteFiles = io.doAs(this::equalityToPosition); + } + + if (!ArrayUtils.isEmpty(input.rewrittenDataFiles())) { + dataFiles = io.doAs(this::rewriterDataFiles); + } + } finally { + dataReader.close(); } + long duration = System.currentTimeMillis() - startTime; + + Map summary = resolverSummary(dataFiles, deleteFiles, duration); + return new RewriteFilesOutput( + dataFiles.toArray(new DataFile[0]), deleteFiles.toArray(new DeleteFile[0]), summary); } finally { - dataReader.close(); + if (shouldClearContext) { + OptimizingTaskLogContext.clearContext(); + } } - long duration = System.currentTimeMillis() - startTime; - - Map summary = resolverSummary(dataFiles, deleteFiles, duration); - return new RewriteFilesOutput( - dataFiles.toArray(new DataFile[0]), deleteFiles.toArray(new DeleteFile[0]), summary); } private List equalityToPosition() throws Exception { diff --git a/amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/DriverLogger.java b/amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/DriverLogger.java deleted file mode 100644 index 0c538b2116..0000000000 --- a/amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/DriverLogger.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.amoro.optimizer.common; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; - -/** Logger that writes driver-level logs to a single file per process ID. */ -public class DriverLogger implements AutoCloseable { - private static final DateTimeFormatter TIMESTAMP_FORMAT = - DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"); - private static final String LOG_BASE_DIR = "/mnt/amoro-logs/compaction"; - - private final long processId; - private final String tableName; - private transient PrintWriter writer; - private transient Path logFile; - - public DriverLogger(long processId, String tableName) { - this.processId = processId; - this.tableName = tableName; - } - - private synchronized void ensureWriter() throws IOException { - if (writer == null) { - Path processDir = Paths.get(LOG_BASE_DIR, String.valueOf(processId)); - Files.createDirectories(processDir); - - logFile = processDir.resolve("driver.log"); - File file = logFile.toFile(); - writer = new PrintWriter(new BufferedWriter(new FileWriter(file, true)), true); - } - } - - public void info(String message, Object... args) { - log("INFO", String.format(message, args), null); - } - - public void warn(String message, Object... args) { - log("WARN", String.format(message, args), null); - } - - public void error(String message, Throwable t) { - log("ERROR", message, t); - } - - public void error(String message, Object... args) { - log("ERROR", String.format(message, args), null); - } - - private synchronized void log(String level, String message, Throwable throwable) { - try { - ensureWriter(); - String timestamp = LocalDateTime.now().format(TIMESTAMP_FORMAT); - String logLine = - String.format("%s %-5s [DRIVER|P:%d] - %s", timestamp, level, processId, message); - writer.println(logLine); - - if (throwable != null) { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - throwable.printStackTrace(pw); - writer.println(sw.toString()); - } - writer.flush(); - } catch (IOException e) { - System.err.println("Failed to write to driver log: " + e.getMessage()); - e.printStackTrace(); - } - } - - @Override - public synchronized void close() { - if (writer != null) { - writer.flush(); - writer.close(); - writer = null; - } - } - - public long getProcessId() { - return processId; - } - - public String getTableName() { - return tableName; - } -} diff --git a/amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/TaskLogger.java b/amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/TaskLogger.java deleted file mode 100644 index bf942441e0..0000000000 --- a/amoro-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/TaskLogger.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.amoro.optimizer.common; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; -import java.io.Serializable; -import java.io.StringWriter; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; - -// writes task-specific logs to separate files organized by process ID -public class TaskLogger implements AutoCloseable, Serializable { - private static final long serialVersionUID = 1L; - private static final DateTimeFormatter TIMESTAMP_FORMAT = - DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"); - private static final String LOG_BASE_DIR = "/mnt/amoro-logs/compaction"; - - private final long processId; - private final int taskId; - private final String tableName; - private transient PrintWriter writer; - private transient Path logFile; - - public TaskLogger(long processId, int taskId, String tableName) { - this.processId = processId; - this.taskId = taskId; - // why do we need tableName, can we please remove it - this.tableName = tableName; - } - - private synchronized void ensureWriter() throws IOException { - if (writer == null) { - Path processDir = Paths.get(LOG_BASE_DIR, String.valueOf(processId)); - Files.createDirectories(processDir); - - logFile = processDir.resolve(taskId + ".log"); - File file = logFile.toFile(); - writer = new PrintWriter(new BufferedWriter(new FileWriter(file, true)), true); - } - } - - public void info(String message, Object... args) { - log("INFO", String.format(message, args), null); - } - - public void warn(String message, Object... args) { - log("WARN", String.format(message, args), null); - } - - public void error(String message, Throwable t) { - log("ERROR", message, t); - } - - public void error(String message, Object... args) { - log("ERROR", String.format(message, args), null); - } - - private synchronized void log(String level, String message, Throwable throwable) { - try { - ensureWriter(); - String timestamp = LocalDateTime.now().format(TIMESTAMP_FORMAT); - String logLine = - String.format("%s %-5s [P:%d|T:%d] - %s", timestamp, level, processId, taskId, message); - writer.println(logLine); - - if (throwable != null) { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - throwable.printStackTrace(pw); - writer.println(sw.toString()); - } - writer.flush(); - } catch (IOException e) { - System.err.println("Failed to write to task log: " + e.getMessage()); - e.printStackTrace(); - } - } - - @Override - public synchronized void close() { - if (writer != null) { - writer.flush(); - writer.close(); - writer = null; - } - } - - public long getProcessId() { - return processId; - } - - public int getTaskId() { - return taskId; - } - - public String getTableName() { - return tableName; - } -} diff --git a/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java b/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java index 1f79476c72..da52ee2694 100644 --- a/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java +++ b/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java @@ -20,10 +20,8 @@ import org.apache.amoro.api.OptimizingTask; import org.apache.amoro.api.OptimizingTaskResult; -import org.apache.amoro.optimizer.common.DriverLogger; import org.apache.amoro.optimizer.common.OptimizerConfig; import org.apache.amoro.optimizer.common.OptimizerExecutor; -import org.apache.amoro.optimizer.common.TaskLogger; import org.apache.amoro.optimizing.RewriteFilesInput; import org.apache.amoro.optimizing.TableOptimizing; import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; @@ -32,6 +30,7 @@ import org.apache.spark.api.java.JavaSparkContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.MDC; import java.util.List; @@ -58,25 +57,26 @@ protected OptimizingTaskResult executeTask(OptimizingTask task) { long processId = task.getTaskId().getProcessId(); int taskId = task.getTaskId().getTaskId(); - String tableName = extractTableName(task); - try (DriverLogger driverLogger = new DriverLogger(processId, tableName); - TaskLogger taskLogger = new TaskLogger(processId, taskId, tableName)) { - driverLogger.info("Starting compaction process"); - driverLogger.info("Process ID: %d, Task ID: %d, Thread: %s", processId, taskId, threadName); + // Set MDC context for Log4j2 routing + // Driver logs go to: //driver.log + // Only set processId (not taskId) — driver handles multiple tasks per process. + MDC.put("processId", String.valueOf(processId)); + MDC.put("logFilePath", processId + "/driver"); + + try { + // LOG.info("Starting task execution"); + // LOG.info("Task ID: {}, Thread: {}", task.getTaskId(), threadName); - taskLogger.info("Starting task execution"); - taskLogger.info("Task ID: %s, Thread: %s", task.getTaskId(), threadName); ImmutableList of = ImmutableList.of(task); jsc.setJobDescription(jobDescription(task)); SparkOptimizingTaskFunction taskFunction = - new SparkOptimizingTaskFunction(getConfig(), threadId, taskLogger); + new SparkOptimizingTaskFunction(getConfig(), threadId); List results = jsc.parallelize(of, 1).map(taskFunction).collect(); result = results.get(0); long duration = System.currentTimeMillis() - startTime; - taskLogger.info("Task completed successfully in %d ms", duration); - driverLogger.info("Task %d completed successfully in %d ms", taskId, duration); + // LOG.info("Task completed successfully in {} ms", duration); LOG.info( "Optimizer executor[{}] executed task[{}] and cost {} ms", threadName, @@ -85,13 +85,7 @@ protected OptimizingTaskResult executeTask(OptimizingTask task) { return result; } catch (Throwable r) { long duration = System.currentTimeMillis() - startTime; - try (DriverLogger errorDriverLogger = new DriverLogger(processId, tableName); - TaskLogger errorLogger = new TaskLogger(processId, taskId, tableName)) { - errorDriverLogger.error("Process failed after %d ms", duration); - errorDriverLogger.error("Error details:", r); - errorLogger.error("Task execution failed after %d ms", duration); - errorLogger.error("Error details:", r); - } + LOG.error("Task execution failed after {} ms", duration); LOG.error( "Optimizer executor[{}] executed task[{}] failed, and cost {} ms", threadName, @@ -101,6 +95,11 @@ protected OptimizingTaskResult executeTask(OptimizingTask task) { result = new OptimizingTaskResult(task.getTaskId(), threadId); result.setErrorMessage(ExceptionUtil.getErrorMessage(r, ERROR_MESSAGE_MAX_LENGTH)); return result; + } finally { + // Do NOT clear MDC here. Keep it set so that the parent class's + // completeTask() and next pollTask()/ackTask() logs also route + // to driver.log instead of the junk ${ctx:logFilePath}.log file. + // MDC will be overwritten at the start of the next executeTask() call. } } @@ -118,17 +117,4 @@ private String jobDescription(OptimizingTask task) { } return description; } - - private String extractTableName(OptimizingTask task) { - try { - TableOptimizing.OptimizingInput input = - SerializationUtil.simpleDeserialize(task.getTaskInput()); - if (input instanceof RewriteFilesInput) { - return ((RewriteFilesInput) input).getTable().name(); - } - } catch (Exception e) { - LOG.warn("Failed to extract table name from task", e); - } - return "unknown"; - } } diff --git a/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizingTaskFunction.java b/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizingTaskFunction.java index 1b251ea5a8..3c347c212a 100644 --- a/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizingTaskFunction.java +++ b/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizingTaskFunction.java @@ -20,12 +20,13 @@ import org.apache.amoro.api.OptimizingTask; import org.apache.amoro.api.OptimizingTaskResult; +import org.apache.amoro.log.OptimizingTaskLogContext; import org.apache.amoro.optimizer.common.OptimizerConfig; import org.apache.amoro.optimizer.common.OptimizerExecutor; -import org.apache.amoro.optimizer.common.TaskLogger; import org.apache.spark.api.java.function.Function; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.MDC; /** * The {@code SparkOptimizingTaskExecuteFunction} defines the whole processing logic that how to @@ -35,30 +36,42 @@ public class SparkOptimizingTaskFunction implements Function//.log + long processId = task.getTaskId().getProcessId(); + int taskId = task.getTaskId().getTaskId(); + String logFilePath = processId + "/" + taskId; + + // Set OptimizingTaskLogContext FIRST so AbstractRewriteFilesExecutor.execute() + // sees isContextSet()==true and does NOT override our MDC with its own format. + OptimizingTaskLogContext.setContext(processId, taskId); + + // Override logFilePath to our desired format + MDC.put("processId", String.valueOf(processId)); + MDC.put("taskId", String.valueOf(taskId)); + MDC.put("logFilePath", logFilePath); + try { + // LOG.info("Executing task on Spark executor"); OptimizingTaskResult result = OptimizerExecutor.executeTask(config, threadId, task, LOG); - if (taskLogger != null) { - taskLogger.info("Task execution completed on executor"); - } + // LOG.info("Task execution completed on executor"); return result; } catch (Exception e) { - if (taskLogger != null) { - taskLogger.error("Task execution failed on executor:", e); - } + LOG.error("Task execution failed on executor", e); throw e; + } finally { + OptimizingTaskLogContext.clearContext(); + MDC.remove("processId"); + MDC.remove("taskId"); + MDC.remove("logFilePath"); } } } diff --git a/dist/src/main/amoro-bin/conf/optimize/log4j2-routing.xml b/dist/src/main/amoro-bin/conf/optimize/log4j2-routing.xml new file mode 100644 index 0000000000..f86464a3f8 --- /dev/null +++ b/dist/src/main/amoro-bin/conf/optimize/log4j2-routing.xml @@ -0,0 +1,132 @@ + + + + + + ${sys:log.home:-logs} + ${env:CONSOLE_LOG_LEVEL:-info} + %d{yyyy-MM-dd HH:mm:ss.SSS} %-5p [%t] [%c{1}] [P:%X{processId}|T:%X{taskId}|Table:%X{tableName}|Exec:%X{executorId}] - %m%n + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docker/amoro/entrypoint.sh b/docker/amoro/entrypoint.sh index e41de1735e..a5d86a746b 100644 --- a/docker/amoro/entrypoint.sh +++ b/docker/amoro/entrypoint.sh @@ -44,9 +44,9 @@ configure_jvm_options() { configure_jvm_options -# creating logs directory with write permissions -mkdir -p /usr/local/amoro/logs/compaction -chmod 777 /usr/local/amoro/logs/compaction +# Create compaction log directory for Log4j2 routing (mounted as shared volume) +mkdir -p /mnt/amoro-logs/compaction +chmod 777 /mnt/amoro-logs/compaction if [ $1 == "help" ]; then printf "Usage: $(basename $0) [ams|optimizer] [args]\n" diff --git a/docker/kind/config.yaml b/docker/kind/config.yaml index 3fce816d9a..49ab6f7ba6 100644 --- a/docker/kind/config.yaml +++ b/docker/kind/config.yaml @@ -176,11 +176,13 @@ containers: spark-conf.spark.kubernetes.driver.secretKeyRef.AWS_SECRET_ACCESS_KEY: "fusion-s3-credentials:secret-access-key" spark-conf.spark.kubernetes.executor.secretKeyRef.AWS_ACCESS_KEY_ID: "fusion-s3-credentials:access-key-id" spark-conf.spark.kubernetes.executor.secretKeyRef.AWS_SECRET_ACCESS_KEY: "fusion-s3-credentials:secret-access-key" - # mounting shared log volume for persistent task logs - # but, why is this required ? + # Mount shared log volume for persistent task logs spark-conf.spark.kubernetes.driver.volumes.hostPath.amoro-logs.mount.path: "/mnt/amoro-logs" spark-conf.spark.kubernetes.driver.volumes.hostPath.amoro-logs.mount.readOnly: "false" spark-conf.spark.kubernetes.driver.volumes.hostPath.amoro-logs.options.path: "/mnt/amoro-logs" spark-conf.spark.kubernetes.executor.volumes.hostPath.amoro-logs.mount.path: "/mnt/amoro-logs" spark-conf.spark.kubernetes.executor.volumes.hostPath.amoro-logs.mount.readOnly: "false" spark-conf.spark.kubernetes.executor.volumes.hostPath.amoro-logs.options.path: "/mnt/amoro-logs" + # Force Log4j2 to use routing config + spark-conf.spark.driver.extraJavaOptions: "-Dlog4j2.configurationFile=file:///opt/spark/conf/log4j2.xml" + spark-conf.spark.executor.extraJavaOptions: "-Dlog4j2.configurationFile=file:///opt/spark/conf/log4j2.xml" diff --git a/docker/kind/docker-compose.yml b/docker/kind/docker-compose.yml index e3d0bdc41c..7c57d4c9be 100644 --- a/docker/kind/docker-compose.yml +++ b/docker/kind/docker-compose.yml @@ -47,6 +47,7 @@ services: - kind-kubeconfig:/root/.kube:ro - spark-home:/opt/spark:ro - amoro-logs:/usr/local/amoro/logs + - amoro-logs:/mnt/amoro-logs command: ["/entrypoint.sh", "ams"] tty: true stdin_open: true diff --git a/docker/optimizer-spark/Dockerfile b/docker/optimizer-spark/Dockerfile index 5365dc48d9..b0b5e59f43 100644 --- a/docker/optimizer-spark/Dockerfile +++ b/docker/optimizer-spark/Dockerfile @@ -17,12 +17,8 @@ # Modified by Datazip Inc. in 2026 ARG SPARK_VERSION=3.5.8 -# Use the explicit java17 tag to guarantee Java 17 runtime. -# apache/spark:3.5.8-java17 and apache/spark:3.5.8 share the same digest today, -# but the plain short tag can resolve to a cached Java 11 layer on Kubernetes nodes. -ARG SPARK_JAVA_TAG=${SPARK_VERSION}-java17 -FROM apache/spark:${SPARK_JAVA_TAG} +FROM apache/spark:${SPARK_VERSION} ARG MAVEN_MIRROR=https://repo.maven.apache.org/maven2 ARG OPTIMIZER_JOB=optimizer-job.jar @@ -31,9 +27,14 @@ ARG TARGETARCH USER root RUN apt-get update \ - && apt-get install -y --no-install-recommends wget \ + && apt-get install -y wget openjdk-17-jre-headless \ && rm -rf /var/lib/apt/lists/* +# Force Spark optimizer runtime to Java 17 and expose a stable JAVA_HOME path. +RUN rm -rf /opt/java/openjdk \ + && mkdir -p /opt/java \ + && ln -sfn /usr/lib/jvm/java-17-openjdk-${TARGETARCH} /opt/java/openjdk \ + && test -x /opt/java/openjdk/bin/java ENV JAVA_HOME=/opt/java/openjdk ENV PATH="${JAVA_HOME}/bin:${PATH}" @@ -43,4 +44,13 @@ RUN cd $SPARK_HOME/jars \ && chown spark:spark *.jar \ && mkdir -p $SPARK_HOME/usrlib +# Remove Spark's default log4j2 config so our XML config takes priority +RUN rm -f $SPARK_HOME/conf/log4j2.properties $SPARK_HOME/conf/log4j2.properties.template + +# Install our log4j2.xml with Routing appender for per-process/task log files +COPY log4j2.xml $SPARK_HOME/conf/log4j2.xml + +# Create the compaction log directory (will be mounted over in K8s) +RUN mkdir -p /mnt/amoro-logs/compaction && chmod 777 /mnt/amoro-logs/compaction + COPY $OPTIMIZER_JOB $SPARK_HOME/usrlib/optimizer-job.jar diff --git a/docker/optimizer-spark/log4j2.xml b/docker/optimizer-spark/log4j2.xml new file mode 100644 index 0000000000..53267f4ce7 --- /dev/null +++ b/docker/optimizer-spark/log4j2.xml @@ -0,0 +1,97 @@ + + + + + + ${env:LOG_DIR:-/mnt/amoro-logs/compaction} + %d{yyyy-MM-dd HH:mm:ss.SSS'Z'}{UTC} %-5p [%t] [%c{1}] [P:%X{processId}|T:%X{taskId}] - %m%n + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 6844dcee083153a57bb4d0e9d0e6b9a3fe2d7e31 Mon Sep 17 00:00:00 2001 From: badalprasadsingh Date: Tue, 17 Mar 2026 09:47:31 +0530 Subject: [PATCH 21/46] add modification header Signed-off-by: badalprasadsingh --- .../java/org/apache/amoro/server/dashboard/DashboardServer.java | 2 ++ .../apache/amoro/server/dashboard/controller/LogController.java | 2 ++ amoro-common/pom.xml | 2 ++ .../java/org/apache/amoro/log/OptimizingTaskLogContext.java | 2 ++ .../apache/amoro/optimizing/AbstractRewriteFilesExecutor.java | 2 ++ .../apache/amoro/optimizer/spark/SparkOptimizerExecutor.java | 2 ++ .../amoro/optimizer/spark/SparkOptimizingTaskFunction.java | 2 ++ dist/src/main/amoro-bin/conf/optimize/log4j2-routing.xml | 2 ++ docker/amoro/entrypoint.sh | 2 ++ docker/optimizer-spark/log4j2.xml | 2 ++ 10 files changed, 20 insertions(+) diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java index ce02bd3f5d..bcadb79c66 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java @@ -14,6 +14,8 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modified by Datazip Inc. in 2026 */ package org.apache.amoro.server.dashboard; diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/LogController.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/LogController.java index 6d6572d156..21358c6e00 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/LogController.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/LogController.java @@ -14,6 +14,8 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modified by Datazip Inc. in 2026 */ package org.apache.amoro.server.dashboard.controller; diff --git a/amoro-common/pom.xml b/amoro-common/pom.xml index 49f0b25198..f08391fd13 100644 --- a/amoro-common/pom.xml +++ b/amoro-common/pom.xml @@ -15,6 +15,8 @@ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ~ See the License for the specific language governing permissions and ~ limitations under the License. + ~ + ~ Modified by Datazip Inc. in 2026 --> diff --git a/amoro-common/src/main/java/org/apache/amoro/log/OptimizingTaskLogContext.java b/amoro-common/src/main/java/org/apache/amoro/log/OptimizingTaskLogContext.java index 5a58949855..8bcd3c463e 100644 --- a/amoro-common/src/main/java/org/apache/amoro/log/OptimizingTaskLogContext.java +++ b/amoro-common/src/main/java/org/apache/amoro/log/OptimizingTaskLogContext.java @@ -14,6 +14,8 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modified by Datazip Inc. in 2026 */ package org.apache.amoro.log; diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/AbstractRewriteFilesExecutor.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/AbstractRewriteFilesExecutor.java index 1b3ba3d960..730132a738 100644 --- a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/AbstractRewriteFilesExecutor.java +++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/AbstractRewriteFilesExecutor.java @@ -14,6 +14,8 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modified by Datazip Inc. in 2026 */ package org.apache.amoro.optimizing; diff --git a/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java b/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java index da52ee2694..f6e42da62c 100644 --- a/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java +++ b/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java @@ -14,6 +14,8 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modified by Datazip Inc. in 2026 */ package org.apache.amoro.optimizer.spark; diff --git a/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizingTaskFunction.java b/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizingTaskFunction.java index 3c347c212a..d21acf79b6 100644 --- a/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizingTaskFunction.java +++ b/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizingTaskFunction.java @@ -14,6 +14,8 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modified by Datazip Inc. in 2026 */ package org.apache.amoro.optimizer.spark; diff --git a/dist/src/main/amoro-bin/conf/optimize/log4j2-routing.xml b/dist/src/main/amoro-bin/conf/optimize/log4j2-routing.xml index f86464a3f8..415ff68e70 100644 --- a/dist/src/main/amoro-bin/conf/optimize/log4j2-routing.xml +++ b/dist/src/main/amoro-bin/conf/optimize/log4j2-routing.xml @@ -15,6 +15,8 @@ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ~ See the License for the specific language governing permissions and ~ limitations under the License. + ~ + ~ Modified by Datazip Inc. in 2026 --> diff --git a/docker/amoro/entrypoint.sh b/docker/amoro/entrypoint.sh index a5d86a746b..f560e7a88f 100644 --- a/docker/amoro/entrypoint.sh +++ b/docker/amoro/entrypoint.sh @@ -16,6 +16,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Modified by Datazip Inc. in 2026 ############################################################################### args=("$@") diff --git a/docker/optimizer-spark/log4j2.xml b/docker/optimizer-spark/log4j2.xml index 53267f4ce7..130d750d56 100644 --- a/docker/optimizer-spark/log4j2.xml +++ b/docker/optimizer-spark/log4j2.xml @@ -15,6 +15,8 @@ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ~ See the License for the specific language governing permissions and ~ limitations under the License. + ~ + ~ Modified by Datazip Inc. in 2026 --> From 3f2b6acbaaef68f1027fbbf87b64f5ece62ca45f Mon Sep 17 00:00:00 2001 From: badalprasadsingh Date: Tue, 17 Mar 2026 19:17:56 +0530 Subject: [PATCH 22/46] add download log APIs Signed-off-by: badalprasadsingh --- amoro-ams/pom.xml | 5 ++ .../server/dashboard/DashboardServer.java | 3 +- .../dashboard/controller/LogController.java | 72 +++++++++++++++++-- .../amoro/log/OptimizingTaskLogContext.java | 3 + .../AbstractRewriteFilesExecutor.java | 53 +++++--------- .../spark/SparkOptimizerExecutor.java | 15 +--- .../spark/SparkOptimizingTaskFunction.java | 2 - docker/kind/docker-compose.yml | 3 +- docker/optimizer-spark/log4j2.xml | 9 ++- 9 files changed, 102 insertions(+), 63 deletions(-) diff --git a/amoro-ams/pom.xml b/amoro-ams/pom.xml index 27eb0800dc..9f745967d3 100644 --- a/amoro-ams/pom.xml +++ b/amoro-ams/pom.xml @@ -455,6 +455,11 @@ ${pagehelper.version} + + org.apache.commons + commons-compress + + org.apache.iceberg iceberg-data diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java index bcadb79c66..2682fa4e82 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java @@ -409,6 +409,8 @@ private EndpointGroup apiGroup() { "/logs", () -> { get("/process/{processId}", logController::getProcessLogs); + get("/process/{processId}/download", logController::downloadProcessLogs); + get("/process/{processId}/file/{fileId}", logController::downloadLogFile); }); }; } @@ -485,7 +487,6 @@ public void handleException(Exception e, Context ctx) { "/swagger-docs", "/api/ams/v1/api/token/calculate/signature", "/api/ams/v1/api/token/calculate/encryptString", - "/api/ams/v1/logs/*", RestCatalogService.ICEBERG_REST_API_PREFIX + "/*" }; diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/LogController.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/LogController.java index 21358c6e00..0d7f251eb2 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/LogController.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/LogController.java @@ -21,11 +21,16 @@ package org.apache.amoro.server.dashboard.controller; import io.javalin.http.Context; +import io.javalin.http.HttpCode; import org.apache.amoro.server.dashboard.response.OkResponse; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; @@ -57,14 +62,13 @@ public void getProcessLogs(Context ctx) { response.put("exists", true); - // Read driver log + // reads driver log Path driverLogPath = processDir.resolve("driver.log"); Map driverLog = new HashMap<>(); if (Files.exists(driverLogPath)) { try { driverLog.put("exists", true); driverLog.put("content", Files.readString(driverLogPath)); - driverLog.put("size", Files.size(driverLogPath)); } catch (IOException e) { LOG.error("Failed to read driver log: {}", driverLogPath, e); driverLog.put("exists", true); @@ -75,13 +79,13 @@ public void getProcessLogs(Context ctx) { } response.put("driverLog", driverLog); - // Read all task logs + // reads all sub-task logs List> taskLogs = new ArrayList<>(); try (DirectoryStream stream = Files.newDirectoryStream(processDir, "*.log")) { for (Path taskLogPath : stream) { String fileName = taskLogPath.getFileName().toString(); if (fileName.equals("driver.log")) { - continue; // Skip driver log + continue; // skip driver log } String taskId = fileName.replace(".log", ""); @@ -91,7 +95,6 @@ public void getProcessLogs(Context ctx) { try { taskLog.put("exists", true); taskLog.put("content", Files.readString(taskLogPath)); - taskLog.put("size", Files.size(taskLogPath)); taskLogs.add(taskLog); } catch (IOException e) { LOG.error("Failed to read task log: {}", taskLogPath, e); @@ -107,4 +110,63 @@ public void getProcessLogs(Context ctx) { response.put("taskLogs", taskLogs); ctx.json(OkResponse.of(response)); } + + public void downloadLogFile(Context ctx) { + String processId = ctx.pathParam("processId"); + String fileId = ctx.pathParam("fileId"); + // fileId is either "driver" or a taskId like "1", "2", etc. + String fileName = fileId + ".log"; + Path logFile = Paths.get(LOG_BASE_DIR, processId, fileName); + + if (!Files.exists(logFile) || !Files.isRegularFile(logFile)) { + ctx.status(HttpCode.NOT_FOUND).result("Log file not found: " + processId + "/" + fileName); + return; + } + + ctx.header("Content-Type", "text/plain; charset=UTF-8"); + ctx.header("Content-Disposition", "attachment; filename=\"" + fileName + "\""); + + try { + ctx.result(Files.newInputStream(logFile)); + } catch (IOException e) { + LOG.error("Failed to read log file: {}", logFile, e); + ctx.status(HttpCode.INTERNAL_SERVER_ERROR).result("Failed to read log file"); + } + } + + public void downloadProcessLogs(Context ctx) { + String processId = ctx.pathParam("processId"); + Path processDir = Paths.get(LOG_BASE_DIR, processId); + + if (!Files.exists(processDir) || !Files.isDirectory(processDir)) { + ctx.status(HttpCode.NOT_FOUND).result("Process log directory not found: " + processId); + return; + } + + String tarFileName = "compaction-logs-" + processId + ".tar"; + ctx.header("Content-Type", "application/x-tar"); + ctx.header("Content-Disposition", "attachment; filename=\"" + tarFileName + "\""); + + try { + OutputStream out = ctx.res.getOutputStream(); + try (TarArchiveOutputStream tar = new TarArchiveOutputStream(new BufferedOutputStream(out))) { + tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); + + try (DirectoryStream stream = Files.newDirectoryStream(processDir, "*.log")) { + for (Path logFile : stream) { + long size = Files.size(logFile); + TarArchiveEntry entry = new TarArchiveEntry(logFile.getFileName().toString()); + entry.setSize(size); + tar.putArchiveEntry(entry); + Files.copy(logFile, tar); + tar.closeArchiveEntry(); + } + } + + tar.finish(); + } + } catch (IOException e) { + LOG.error("Failed to create tar archive for process {}", processId, e); + } + } } diff --git a/amoro-common/src/main/java/org/apache/amoro/log/OptimizingTaskLogContext.java b/amoro-common/src/main/java/org/apache/amoro/log/OptimizingTaskLogContext.java index 8bcd3c463e..8a94c74e71 100644 --- a/amoro-common/src/main/java/org/apache/amoro/log/OptimizingTaskLogContext.java +++ b/amoro-common/src/main/java/org/apache/amoro/log/OptimizingTaskLogContext.java @@ -22,12 +22,15 @@ import org.apache.logging.log4j.ThreadContext; +// manages per-thread logging context using Log4j2 thread context: setting "processId" and "taskId" +// isContextSet: prevent overwriting an already-initialized MDC context public class OptimizingTaskLogContext { public static final String PROCESS_ID_KEY = "processId"; public static final String TASK_ID_KEY = "taskId"; public static final String LOG_FILE_PATH_KEY = "logFilePath"; + // per-thread flag indicating whether a caller has already set up logging context. private static final ThreadLocal CONTEXT_SET = ThreadLocal.withInitial(() -> false); public static void setContext(long processId, int taskId) { diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/AbstractRewriteFilesExecutor.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/AbstractRewriteFilesExecutor.java index 730132a738..53bd71c929 100644 --- a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/AbstractRewriteFilesExecutor.java +++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/AbstractRewriteFilesExecutor.java @@ -27,7 +27,6 @@ import org.apache.amoro.data.DataTreeNode; import org.apache.amoro.io.AuthenticatedFileIO; import org.apache.amoro.io.writer.SetTreeNode; -import org.apache.amoro.log.OptimizingTaskLogContext; import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; import org.apache.amoro.table.MixedTable; import org.apache.amoro.table.TableProperties; @@ -99,48 +98,28 @@ public AbstractRewriteFilesExecutor( @Override public RewriteFilesOutput execute() { - boolean shouldClearContext = false; - if (!OptimizingTaskLogContext.isContextSet() - && properties.containsKey(TaskProperties.PROCESS_ID)) { - try { - long processId = Long.parseLong(properties.get(TaskProperties.PROCESS_ID)); - int taskId = - properties.containsKey("taskId") ? Integer.parseInt(properties.get("taskId")) : -1; - OptimizingTaskLogContext.setContext(processId, taskId); - shouldClearContext = true; - } catch (Exception e) { - LOG.warn("Failed to set logging context in AbstractRewriteFilesExecutor", e); - } - } - - try { - LOG.info("Start processing table optimize task: {}", input); - - List dataFiles = new ArrayList<>(); - List deleteFiles = new ArrayList<>(); + LOG.info("Start processing table optimize task: {}", input); - long startTime = System.currentTimeMillis(); - try { - if (!ArrayUtils.isEmpty(input.rePosDeletedDataFiles())) { - deleteFiles = io.doAs(this::equalityToPosition); - } + List dataFiles = new ArrayList<>(); + List deleteFiles = new ArrayList<>(); - if (!ArrayUtils.isEmpty(input.rewrittenDataFiles())) { - dataFiles = io.doAs(this::rewriterDataFiles); - } - } finally { - dataReader.close(); + long startTime = System.currentTimeMillis(); + try { + if (!ArrayUtils.isEmpty(input.rePosDeletedDataFiles())) { + deleteFiles = io.doAs(this::equalityToPosition); } - long duration = System.currentTimeMillis() - startTime; - Map summary = resolverSummary(dataFiles, deleteFiles, duration); - return new RewriteFilesOutput( - dataFiles.toArray(new DataFile[0]), deleteFiles.toArray(new DeleteFile[0]), summary); - } finally { - if (shouldClearContext) { - OptimizingTaskLogContext.clearContext(); + if (!ArrayUtils.isEmpty(input.rewrittenDataFiles())) { + dataFiles = io.doAs(this::rewriterDataFiles); } + } finally { + dataReader.close(); } + long duration = System.currentTimeMillis() - startTime; + + Map summary = resolverSummary(dataFiles, deleteFiles, duration); + return new RewriteFilesOutput( + dataFiles.toArray(new DataFile[0]), deleteFiles.toArray(new DeleteFile[0]), summary); } private List equalityToPosition() throws Exception { diff --git a/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java b/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java index f6e42da62c..5ee52bd220 100644 --- a/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java +++ b/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java @@ -62,28 +62,21 @@ protected OptimizingTaskResult executeTask(OptimizingTask task) { // Set MDC context for Log4j2 routing // Driver logs go to: //driver.log - // Only set processId (not taskId) — driver handles multiple tasks per process. MDC.put("processId", String.valueOf(processId)); MDC.put("logFilePath", processId + "/driver"); try { - // LOG.info("Starting task execution"); - // LOG.info("Task ID: {}, Thread: {}", task.getTaskId(), threadName); - ImmutableList of = ImmutableList.of(task); jsc.setJobDescription(jobDescription(task)); SparkOptimizingTaskFunction taskFunction = new SparkOptimizingTaskFunction(getConfig(), threadId); List results = jsc.parallelize(of, 1).map(taskFunction).collect(); result = results.get(0); - - long duration = System.currentTimeMillis() - startTime; - // LOG.info("Task completed successfully in {} ms", duration); LOG.info( "Optimizer executor[{}] executed task[{}] and cost {} ms", threadName, task.getTaskId(), - duration); + System.currentTimeMillis() - startTime); return result; } catch (Throwable r) { long duration = System.currentTimeMillis() - startTime; @@ -98,10 +91,8 @@ protected OptimizingTaskResult executeTask(OptimizingTask task) { result.setErrorMessage(ExceptionUtil.getErrorMessage(r, ERROR_MESSAGE_MAX_LENGTH)); return result; } finally { - // Do NOT clear MDC here. Keep it set so that the parent class's - // completeTask() and next pollTask()/ackTask() logs also route - // to driver.log instead of the junk ${ctx:logFilePath}.log file. - // MDC will be overwritten at the start of the next executeTask() call. + MDC.remove("processId"); + MDC.remove("logFilePath"); } } diff --git a/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizingTaskFunction.java b/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizingTaskFunction.java index d21acf79b6..17307c14d6 100644 --- a/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizingTaskFunction.java +++ b/amoro-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizingTaskFunction.java @@ -62,9 +62,7 @@ public OptimizingTaskResult call(OptimizingTask task) { MDC.put("logFilePath", logFilePath); try { - // LOG.info("Executing task on Spark executor"); OptimizingTaskResult result = OptimizerExecutor.executeTask(config, threadId, task, LOG); - // LOG.info("Task execution completed on executor"); return result; } catch (Exception e) { LOG.error("Task execution failed on executor", e); diff --git a/docker/kind/docker-compose.yml b/docker/kind/docker-compose.yml index 7c57d4c9be..a891b399c7 100644 --- a/docker/kind/docker-compose.yml +++ b/docker/kind/docker-compose.yml @@ -381,4 +381,5 @@ networks: volumes: kind-kubeconfig: spark-home: - amoro-logs: \ No newline at end of file + amoro-logs: + \ No newline at end of file diff --git a/docker/optimizer-spark/log4j2.xml b/docker/optimizer-spark/log4j2.xml index 130d750d56..09016a4c40 100644 --- a/docker/optimizer-spark/log4j2.xml +++ b/docker/optimizer-spark/log4j2.xml @@ -15,19 +15,18 @@ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ~ See the License for the specific language governing permissions and ~ limitations under the License. - ~ - ~ Modified by Datazip Inc. in 2026 --> ${env:LOG_DIR:-/mnt/amoro-logs/compaction} - %d{yyyy-MM-dd HH:mm:ss.SSS'Z'}{UTC} %-5p [%t] [%c{1}] [P:%X{processId}|T:%X{taskId}] - %m%n + %d{yyyy-MM-dd HH:mm:ss.SSS'Z'}{UTC} %-5p [%t] [%c{1}] [P:%X{processId}|T:%X{taskId}] - %m%n + {"level":"%p","time":"%d{yyyy-MM-dd'T'HH:mm:ss.SSS'Z'}{UTC}","processId":"%X{processId}","taskId":"%X{taskId}","logger":"%c{1}","message":"%enc{%m}{JSON}"}%n - + @@ -44,7 +43,7 @@ createOnDemand="true" bufferedIO="false" immediateFlush="true"> - + From 55b56d695b8ebb3f146a451d385e7ef94afcca2d Mon Sep 17 00:00:00 2001 From: badalprasadsingh Date: Tue, 17 Mar 2026 22:10:16 +0530 Subject: [PATCH 23/46] fix: minor Signed-off-by: badalprasadsingh --- .../apache/amoro/optimizing/AbstractRewriteFilesExecutor.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/AbstractRewriteFilesExecutor.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/AbstractRewriteFilesExecutor.java index 53bd71c929..4446619f92 100644 --- a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/AbstractRewriteFilesExecutor.java +++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/AbstractRewriteFilesExecutor.java @@ -14,8 +14,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * - * Modified by Datazip Inc. in 2026 */ package org.apache.amoro.optimizing; From 17fddfaebb44fee626ef45c518329e8a588b1504 Mon Sep 17 00:00:00 2001 From: hashcode-ankit Date: Fri, 20 Mar 2026 15:47:28 +0530 Subject: [PATCH 24/46] feat: migrated interval to cron based configuration --- .../amoro/server/AmoroManagementConf.java | 2 +- .../server/DefaultOptimizingService.java | 7 +- .../server/optimizing/OptimizingQueue.java | 42 +++- .../server/optimizing/SchedulingPolicy.java | 61 ++++- .../inline/TableRuntimeRefreshExecutor.java | 219 ++++++++++++----- .../server/table/DefaultTableRuntime.java | 138 ++++++++++- .../server/table/TableConfigurations.java | 26 +- .../optimizing/TestMixedHiveOptimizing.java | 2 +- .../TestMixedIcebergOptimizing.java | 4 +- .../optimizing/TestOptimizingIntegration.java | 4 +- .../optimizing/TestOptimizingQueue.java | 2 +- .../flow/TestKeyedContinuousOptimizing.java | 8 +- .../flow/TestUnKeyedContinuousOptimizing.java | 8 +- .../plan/MixedTablePlanTestBase.java | 6 +- .../apache/amoro/config/OptimizingConfig.java | 68 +++--- .../apache/amoro/process/ProcessStatus.java | 4 +- .../org/apache/amoro/utils/CronUtils.java | 229 ++++++++++++++++++ .../TableRuntimeOptimizingState.java | 14 ++ .../plan/CommonPartitionEvaluator.java | 28 ++- .../apache/amoro/table/TableProperties.java | 31 ++- .../TestMetadataBasedEvaluationEvent.java | 2 - .../amoro/hive/catalog/MixedHiveTables.java | 5 - .../views/tables/components/Optimizing.vue | 3 +- 23 files changed, 743 insertions(+), 170 deletions(-) create mode 100644 amoro-common/src/main/java/org/apache/amoro/utils/CronUtils.java diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java index 96e54b7df9..a1f30716f2 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java @@ -95,7 +95,7 @@ public class AmoroManagementConf { public static final ConfigOption REFRESH_EXTERNAL_CATALOGS_INTERVAL = ConfigOptions.key("refresh-external-catalogs.interval") .durationType() - .defaultValue(Duration.ofMinutes(3)) + .defaultValue(Duration.ofMinutes(1)) .withDescription("Interval to refresh the external catalog."); public static final ConfigOption REFRESH_EXTERNAL_CATALOGS_THREAD_COUNT = diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/DefaultOptimizingService.java b/amoro-ams/src/main/java/org/apache/amoro/server/DefaultOptimizingService.java index f3b13e69a6..92fac15f3d 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/DefaultOptimizingService.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/DefaultOptimizingService.java @@ -369,7 +369,12 @@ private class TableRuntimeHandlerImpl extends RuntimeHandlerChain { @Override public void handleStatusChanged(TableRuntime tableRuntime, OptimizingStatus originalStatus) { DefaultTableRuntime defaultTableRuntime = (DefaultTableRuntime) tableRuntime; - if (!defaultTableRuntime.getOptimizingStatus().isProcessing()) { + OptimizingStatus newStatus = defaultTableRuntime.getOptimizingStatus(); + // Only re-queue when the table becomes available for the next planning cycle (IDLE after + // completing/skipping work) or has new pending data (PENDING). Skip the IDLE→PLANNING + // transition to avoid a redundant "Bind queue" log and unnecessary scheduler churn while + // planning is already in flight. + if (newStatus == OptimizingStatus.IDLE || newStatus == OptimizingStatus.PENDING) { getOptionalQueueByGroup(defaultTableRuntime.getGroupName()) .ifPresent(q -> q.refreshTable(defaultTableRuntime)); } diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/OptimizingQueue.java b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/OptimizingQueue.java index 964d27c7b8..090f824d4c 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/OptimizingQueue.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/OptimizingQueue.java @@ -156,6 +156,18 @@ private void initTableRuntime(DefaultTableRuntime tableRuntime) { process.close(false); } } + // A FAILED process cannot dispatch tasks (poll() returns null when status=FAILED), + // so it would never trigger acceptResult/persistAndSetCompleted and the table would + // stay stuck in a processing status forever. Reset the table to IDLE immediately. + if (process != null && process.getStatus() == ProcessStatus.FAILED) { + LOG.warn( + "Recovering table {} with a FAILED process {} — resetting to IDLE so scheduling" + + " can resume", + tableRuntime.getTableIdentifier(), + process.getProcessId()); + tableRuntime.completeProcess(false); + process = null; + } if (!tableRuntime.getOptimizingStatus().isProcessing()) { scheduler.addTable(tableRuntime); } else if (process != null) { @@ -653,6 +665,21 @@ private void acceptResult(TaskRuntime taskRuntime) { private void resetTask(TaskRuntime taskRuntime) { lock.lock(); try { + // If the task was actively executing (SCHEDULED or ACKED), the slot it held in + // optimizingTasksMap was incremented by poll() but acceptResult() was never called + // (the optimizer died). Decrement now so the quota count stays accurate and the + // next poll isn't incorrectly blocked by a phantom quota entry. + if (taskRuntime.getStatus() == TaskRuntime.Status.SCHEDULED + || taskRuntime.getStatus() == TaskRuntime.Status.ACKED) { + optimizingTasksMap.computeIfPresent( + tableRuntime.getTableIdentifier(), + (k, v) -> { + if (v.get() > 0) { + v.decrementAndGet(); + } + return v; + }); + } taskRuntime.reset(); taskQueue.add(taskRuntime); } finally { @@ -914,7 +941,20 @@ private void loadTaskRuntimes(OptimizingProcess optimizingProcess) { if (taskRuntime.getStatus() == TaskRuntime.Status.PLANNED) { taskQueue.offer(taskRuntime); } else if (taskRuntime.getStatus() == TaskRuntime.Status.FAILED) { - retryTask(taskRuntime); + taskRuntime.reset(); + taskQueue.offer(taskRuntime); + } else if (taskRuntime.getStatus() == TaskRuntime.Status.SCHEDULED + || taskRuntime.getStatus() == TaskRuntime.Status.ACKED) { + // After a server restart the optimizer that held this task is gone. + // Reset immediately so any reconnecting optimizer can pick it up rather + // than waiting for the OptimizerKeeper's heartbeat expiry cycle (which + // requires an optimizer to be connected and its timer to fire first). + LOG.warn( + "Resetting {} task {} on recovery — optimizer token is stale after restart", + taskRuntime.getStatus(), + taskRuntime.getTaskId()); + taskRuntime.reset(); + taskQueue.offer(taskRuntime); } }); } catch (IllegalArgumentException e) { diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/SchedulingPolicy.java b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/SchedulingPolicy.java index 0759f1e367..b55b39aef9 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/SchedulingPolicy.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/SchedulingPolicy.java @@ -39,6 +39,21 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +/** + * Selects the next table to plan from the set of tables currently registered in an optimizer group. + * + *

A table is eligible for planning if and only if: + * + *

    + *
  • It is not already being processed (FULL/MAJOR/MINOR_OPTIMIZING, COMMITTING) or planned. + *
  • Its status is {@link OptimizingStatus#PENDING} — meaning the cron-tick scheduler in + * {@code TableRuntimeRefreshExecutor} already determined that a cron expression fired and + * the optimization is necessary. + *
+ * + *

Cron evaluation, "not necessary" skip logic, and SKIPPED record creation are all handled + * upstream in {@code TableRuntimeRefreshExecutor}. This class is intentionally kept simple. + */ public class SchedulingPolicy { public static final Logger LOG = LoggerFactory.getLogger(SchedulingPolicy.class); @@ -106,22 +121,46 @@ private Comparator createSorterByPolicy() { } } + /** + * Populates {@code originalSet} with the identifiers of tables that are not eligible for + * scheduling right now. A table is ineligible if it is already being processed / planned, or if + * it has not been marked PENDING by the cron-tick scheduler. + */ private void fillSkipSet(Set originalSet) { - long currentTime = System.currentTimeMillis(); tableRuntimeMap.values().stream() - .filter( - tableRuntime -> - !isTablePending(tableRuntime) - || currentTime - tableRuntime.getLastPlanTime() - < tableRuntime.getOptimizingConfig().getMinPlanInterval()) + .filter(tableRuntime -> !isEligibleForScheduling(tableRuntime)) .forEach(tableRuntime -> originalSet.add(tableRuntime.getTableIdentifier())); } - private boolean isTablePending(DefaultTableRuntime tableRuntime) { - return tableRuntime.getOptimizingStatus() == OptimizingStatus.PENDING - && (tableRuntime.getLastOptimizedSnapshotId() != tableRuntime.getCurrentSnapshotId() - || tableRuntime.getLastOptimizedChangeSnapshotId() - != tableRuntime.getCurrentChangeSnapshotId()); + /** + * A table is eligible for scheduling when: + * + *

    + *
  1. It is not already running or being planned (no concurrent processing). + *
  2. Its status is {@link OptimizingStatus#PENDING} — set exclusively by the cron-tick + * scheduler after verifying that a cron expression fired and the optimization is necessary. + *
+ */ + private boolean isEligibleForScheduling(DefaultTableRuntime tableRuntime) { + OptimizingStatus status = tableRuntime.getOptimizingStatus(); + + if (status.isProcessing() || status == OptimizingStatus.PLANNING) { + LOG.debug( + "[optimization-skip] {} skipped - already {} (will schedule after completion)", + tableRuntime.getTableIdentifier(), + status.displayValue()); + return false; + } + + if (status != OptimizingStatus.PENDING) { + LOG.debug( + "[optimization-skip] {} skipped - status is {} (waiting for cron tick to set PENDING)", + tableRuntime.getTableIdentifier(), + status.displayValue()); + return false; + } + + return true; } public void addTable(DefaultTableRuntime tableRuntime) { diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/TableRuntimeRefreshExecutor.java b/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/TableRuntimeRefreshExecutor.java index 18071a2cdd..61ee66d0da 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/TableRuntimeRefreshExecutor.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/scheduler/inline/TableRuntimeRefreshExecutor.java @@ -22,30 +22,48 @@ import org.apache.amoro.TableRuntime; import org.apache.amoro.config.OptimizingConfig; import org.apache.amoro.config.TableConfiguration; -import org.apache.amoro.optimizing.evaluation.MetadataBasedEvaluationEvent; -import org.apache.amoro.optimizing.plan.AbstractOptimizingEvaluator; +import org.apache.amoro.optimizing.OptimizingType; import org.apache.amoro.process.ProcessStatus; import org.apache.amoro.server.optimizing.OptimizingProcess; import org.apache.amoro.server.optimizing.OptimizingStatus; import org.apache.amoro.server.scheduler.PeriodicTableScheduler; import org.apache.amoro.server.table.DefaultTableRuntime; import org.apache.amoro.server.table.TableService; -import org.apache.amoro.server.utils.IcebergTableUtil; import org.apache.amoro.shade.guava32.com.google.common.base.Preconditions; import org.apache.amoro.table.MixedTable; +import org.apache.amoro.utils.CronUtils; -/** Executor that refreshes table runtimes and evaluates optimizing status periodically. */ +/** + * Minute-tick scheduler that drives all self-optimizing purely through cron expressions. + * + *

Each tick (default: every 1 minute) it: + * + *

    + *
  1. Refreshes the table's snapshot metadata. + *
  2. Evaluates full → major → minor cron expressions in priority order. + *
  3. For each fired cron, checks whether the optimization is still "necessary": + *
      + *
    • A type is unnecessary when the snapshot has not changed since the last + * optimization AND the last optimization type already covers this type + * (FULL covers all; MAJOR covers major + minor; MINOR covers only minor). + *
    + *
  4. If necessary → transitions the table to PENDING so {@code OptimizingQueue} will plan it. + *
  5. If unnecessary → writes a {@link ProcessStatus#SKIPPED} record and falls through to the + * next lower-priority type. + *
+ * + *

Snapshot changes do not trigger optimization on their own; every optimization cycle + * must be backed by a cron expression that has fired since the last run of that type. + */ public class TableRuntimeRefreshExecutor extends PeriodicTableScheduler { - // 1 minutes + /** How long to wait between consecutive ticks for the same table (default 1 minute). */ private final long interval; - private final int maxPendingPartitions; public TableRuntimeRefreshExecutor( TableService tableService, int poolSize, long interval, int maxPendingPartitions) { super(tableService, poolSize); this.interval = interval; - this.maxPendingPartitions = maxPendingPartitions; } @Override @@ -55,48 +73,18 @@ protected boolean enabled(TableRuntime tableRuntime) { @Override protected long getNextExecutingTime(TableRuntime tableRuntime) { - DefaultTableRuntime defaultTableRuntime = (DefaultTableRuntime) tableRuntime; - return Math.min( - defaultTableRuntime.getOptimizingConfig().getMinorLeastInterval() * 4L / 5, interval); + return interval; } - private void tryEvaluatingPendingInput(DefaultTableRuntime tableRuntime, MixedTable table) { - // only evaluate pending input when optimizing is enabled and in idle state - OptimizingConfig optimizingConfig = tableRuntime.getOptimizingConfig(); - if (optimizingConfig.isEnabled() - && tableRuntime.getOptimizingStatus().equals(OptimizingStatus.IDLE)) { - - if (optimizingConfig.isMetadataBasedTriggerEnabled() - && !MetadataBasedEvaluationEvent.isEvaluatingNecessary( - optimizingConfig, table, tableRuntime.getLastPlanTime())) { - logger.debug( - "{} optimizing is not necessary due to metadata based trigger", - tableRuntime.getTableIdentifier()); - return; - } - - AbstractOptimizingEvaluator evaluator = - IcebergTableUtil.createOptimizingEvaluator(tableRuntime, table, maxPendingPartitions); - if (evaluator.isNecessary()) { - AbstractOptimizingEvaluator.PendingInput pendingInput = - evaluator.getOptimizingPendingInput(); - logger.debug( - "{} optimizing is necessary and get pending input {}", - tableRuntime.getTableIdentifier(), - pendingInput); - tableRuntime.setPendingInput(pendingInput); - } else { - tableRuntime.optimizingNotNecessary(); - } - tableRuntime.setTableSummary(evaluator.getPendingInput()); - } + @Override + protected long getExecutorDelay() { + return 0; } @Override public void handleConfigChanged(TableRuntime tableRuntime, TableConfiguration originalConfig) { Preconditions.checkArgument(tableRuntime instanceof DefaultTableRuntime); DefaultTableRuntime defaultTableRuntime = (DefaultTableRuntime) tableRuntime; - // After disabling self-optimizing, close the currently running optimizing process. if (originalConfig.getOptimizingConfig().isEnabled() && !tableRuntime.getTableConfiguration().getOptimizingConfig().isEnabled()) { OptimizingProcess optimizingProcess = defaultTableRuntime.getOptimizingProcess(); @@ -106,32 +94,149 @@ public void handleConfigChanged(TableRuntime tableRuntime, TableConfiguration or } } - @Override - protected long getExecutorDelay() { - return 0; - } - @Override public void execute(TableRuntime tableRuntime) { try { Preconditions.checkArgument(tableRuntime instanceof DefaultTableRuntime); DefaultTableRuntime defaultTableRuntime = (DefaultTableRuntime) tableRuntime; - long lastOptimizedSnapshotId = defaultTableRuntime.getLastOptimizedSnapshotId(); - long lastOptimizedChangeSnapshotId = defaultTableRuntime.getLastOptimizedChangeSnapshotId(); AmoroTable table = loadTable(tableRuntime); defaultTableRuntime.refresh(table); - MixedTable mixedTable = (MixedTable) table.originalTable(); - if ((mixedTable.isKeyedTable() - && (lastOptimizedSnapshotId != defaultTableRuntime.getCurrentSnapshotId() - || lastOptimizedChangeSnapshotId - != defaultTableRuntime.getCurrentChangeSnapshotId())) - || (mixedTable.isUnkeyedTable() - && lastOptimizedSnapshotId != defaultTableRuntime.getCurrentSnapshotId())) { - tryEvaluatingPendingInput(defaultTableRuntime, mixedTable); - } + + evaluateCronTriggers(defaultTableRuntime, (MixedTable) table.originalTable()); } catch (Throwable throwable) { logger.error("Refreshing table {} failed.", tableRuntime.getTableIdentifier(), throwable); } } + + /** + * Core cron-tick logic. Iterates FULL → MAJOR → MINOR in priority order. The first type whose + * cron has fired AND whose optimization is still necessary transitions the table to PENDING and + * stops. Types whose crons fired but are unnecessary get a SKIPPED process record and the loop + * falls through to the next type. + */ + private void evaluateCronTriggers(DefaultTableRuntime tableRuntime, MixedTable mixedTable) { + OptimizingConfig cfg = tableRuntime.getOptimizingConfig(); + if (!cfg.isEnabled()) { + return; + } + + // Only consider tables that are idle; already-processing / planning / pending tables + // will be handled by the existing OptimizingQueue machinery. + OptimizingStatus status = tableRuntime.getOptimizingStatus(); + if (status != OptimizingStatus.IDLE) { + return; + } + + long now = System.currentTimeMillis(); + boolean snapshotChanged = isSnapshotChanged(tableRuntime, mixedTable); + OptimizingType lastType = tableRuntime.getLastOptimizingType(); + + boolean scheduled = false; + OptimizingType scheduledType = null; + + for (OptimizingType candidate : new OptimizingType[] { + OptimizingType.FULL, OptimizingType.MAJOR, OptimizingType.MINOR}) { + + String cronExpr = cronExpressionFor(cfg, candidate); + long lastOptimizingTime = lastOptimizingTimeFor(tableRuntime, candidate); + + if (!CronUtils.hasFiredSince(cronExpr, lastOptimizingTime, now)) { + // Cron has not fired since the last run of this type — skip silently. + continue; + } + + if (!scheduled && (snapshotChanged || isNecessary(candidate, lastType))) { + // Cron fired AND optimization is necessary — mark the table as pending. + logger.info( + "[cron-trigger] table={} scheduling {} optimization (snapshotChanged={}, lastType={})", + tableRuntime.getTableIdentifier(), + candidate, + snapshotChanged, + lastType); + tableRuntime.markAsPending(candidate); + scheduled = true; + scheduledType = candidate; + continue; + } + + // Cron fired but there is nothing new to optimize at this level, or a higher priority optimization is already scheduled. + String reason; + if (scheduled) { + reason = String.format("skipped because higher priority %s optimization is scheduled", scheduledType); + } else { + reason = buildSkipReason(candidate, lastType); + } + logger.info( + "[cron-skip] table={} type={} skipped: {}", + tableRuntime.getTableIdentifier(), + candidate, + reason); + tableRuntime.recordSkippedOptimization(candidate, reason); + } + } + + // ── helpers ───────────────────────────────────────────────────────────────── + + /** + * Returns {@code true} when the table's current snapshot differs from the snapshot that was + * current when the last optimization completed. + */ + private boolean isSnapshotChanged(DefaultTableRuntime tableRuntime, MixedTable mixedTable) { + long lastOptSnapshotId = tableRuntime.getLastOptimizedSnapshotId(); + long lastOptChangeSnapshotId = tableRuntime.getLastOptimizedChangeSnapshotId(); + if (mixedTable.isKeyedTable()) { + return lastOptSnapshotId != tableRuntime.getCurrentSnapshotId() + || lastOptChangeSnapshotId != tableRuntime.getCurrentChangeSnapshotId(); + } + return lastOptSnapshotId != tableRuntime.getCurrentSnapshotId(); + } + + /** + * A type is "unnecessary" when the snapshot is unchanged AND the last completed optimization + * already covers this type: + * + *

    + *
  • FULL is covered only by a previous FULL. + *
  • MAJOR is covered by a previous MAJOR or FULL. + *
  • MINOR is covered by any previous optimization (MINOR, MAJOR, or FULL). + *
+ */ + private boolean isNecessary(OptimizingType candidate, OptimizingType lastType) { + if (lastType == null) { + return false; + } + switch (candidate) { + case FULL: + return lastType == OptimizingType.MINOR || lastType == OptimizingType.MAJOR; + case MAJOR: + return lastType == OptimizingType.MINOR; + default: + return false; + } + } + + private String buildSkipReason(OptimizingType candidate, OptimizingType lastType) { + return String.format( + "cron fired for %s but snapshot unchanged since last %s optimization — no new data to process", + candidate, lastType); + } + + private String cronExpressionFor(OptimizingConfig cfg, OptimizingType type) { + switch (type) { + case FULL: return cfg.getFullTriggerCron(); + case MAJOR: return cfg.getMajorTriggerCron(); + case MINOR: return cfg.getMinorTriggerCron(); + default: return null; + } + } + + private long lastOptimizingTimeFor(DefaultTableRuntime tableRuntime, OptimizingType type) { + switch (type) { + case FULL: return tableRuntime.getLastFullOptimizingTime(); + case MAJOR: return tableRuntime.getLastMajorOptimizingTime(); + case MINOR: return tableRuntime.getLastMinorOptimizingTime(); + default: return 0L; + } + } } diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultTableRuntime.java b/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultTableRuntime.java index 11c1a21cff..1720f10bba 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultTableRuntime.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/table/DefaultTableRuntime.java @@ -41,6 +41,7 @@ import org.apache.amoro.server.persistence.mapper.OptimizerMapper; import org.apache.amoro.server.persistence.mapper.OptimizingProcessMapper; import org.apache.amoro.server.persistence.mapper.TableBlockerMapper; +import org.apache.amoro.server.persistence.mapper.TableProcessMapper; import org.apache.amoro.server.resource.OptimizerInstance; import org.apache.amoro.server.table.blocker.TableBlocker; import org.apache.amoro.server.table.cleanup.CleanupOperation; @@ -102,6 +103,7 @@ public class DefaultTableRuntime extends AbstractTableRuntime private final TableSummaryMetrics tableSummaryMetrics; private volatile long lastPlanTime; private volatile OptimizingProcess optimizingProcess; + private volatile OptimizingType pendingCronType; private final List taskQuotas = new CopyOnWriteArrayList<>(); public DefaultTableRuntime(TableRuntimeStore store) { @@ -201,6 +203,22 @@ public long getLastMinorOptimizingTime() { return store().getState(OPTIMIZING_STATE_KEY).getLastMinorOptimizingTime(); } + /** + * Returns the type of the last successfully completed optimization, or {@code null} if none has + * ever run. Used by the cron-tick scheduler to determine which types are still "necessary". + */ + public OptimizingType getLastOptimizingType() { + String raw = store().getState(OPTIMIZING_STATE_KEY).getLastOptimizingType(); + if (raw == null || raw.isEmpty()) { + return null; + } + try { + return OptimizingType.valueOf(raw); + } catch (IllegalArgumentException e) { + return null; + } + } + public long getLastOptimizedChangeSnapshotId() { return store().getState(OPTIMIZING_STATE_KEY).getLastOptimizedChangeSnapshotId(); } @@ -414,8 +432,19 @@ public void completeProcess(boolean success) { .updateState( OPTIMIZING_STATE_KEY, state -> { - state.setLastOptimizedSnapshotId(optimizingProcess.getTargetSnapshotId()); - state.setLastOptimizedChangeSnapshotId(optimizingProcess.getTargetChangeSnapshotId()); + if (success) { + // Only advance the snapshot checkpoints on success. Leaving them at their previous + // values on failure ensures the next cron tick sees a snapshot change and + // re-schedules the optimization instead of treating the table as already optimized. + state.setLastOptimizedSnapshotId(optimizingProcess.getTargetSnapshotId()); + state.setLastOptimizedChangeSnapshotId( + optimizingProcess.getTargetChangeSnapshotId()); + state.setLastOptimizingType(processType.name()); + } + // Always advance the per-type timing regardless of success/failure. + // This ensures the cron interval acts as a natural retry backoff: without this, a + // failed optimization would leave lastXxxOptimizingTime at 0, causing hasFiredSince() + // to return true on every 1-minute tick and rescheduling the optimization every minute. if (processType == OptimizingType.MINOR) { state.setLastMinorOptimizingTime(optimizingProcess.getPlanTime()); } else if (processType == OptimizingType.MAJOR) { @@ -430,6 +459,7 @@ public void completeProcess(boolean success) { optimizingMetrics.processComplete(processType, success, optimizingProcess.getPlanTime()); optimizingProcess = null; + this.pendingCronType = null; } public void completeEmptyProcess() { @@ -437,6 +467,8 @@ public void completeEmptyProcess() { boolean needUpdate = originalStatus == OptimizingStatus.PLANNING || originalStatus == OptimizingStatus.PENDING; if (needUpdate) { + OptimizingType cronType = this.pendingCronType; + long now = System.currentTimeMillis(); store() .begin() .updateStatusCode(code -> OptimizingStatus.IDLE.getCode()) @@ -445,10 +477,21 @@ public void completeEmptyProcess() { state -> { state.setLastOptimizedSnapshotId(state.getCurrentSnapshotId()); state.setLastOptimizedChangeSnapshotId(state.getCurrentChangeSnapshotId()); + if (cronType != null) { + if (cronType == OptimizingType.MINOR) { + state.setLastMinorOptimizingTime(now); + } else if (cronType == OptimizingType.MAJOR) { + state.setLastMajorOptimizingTime(now); + } else if (cronType == OptimizingType.FULL) { + state.setLastFullOptimizingTime(now); + } + state.setLastOptimizingType(cronType.name()); + } return state; }) .updateState(PENDING_INPUT_KEY, any -> new AbstractOptimizingEvaluator.PendingInput()) .commit(); + this.pendingCronType = null; } } @@ -467,6 +510,97 @@ public void optimizingNotNecessary() { } } + /** + * Transitions this table from IDLE to PENDING without performing a file scan. Used by the + * cron-tick scheduler after determining that an optimization type is eligible and necessary. + * The actual file analysis is deferred to the planner inside {@code OptimizingQueue.planInternal}. + * + * @param cronType the optimization type whose cron triggered this transition — stored so that + * {@link #completeEmptyProcess()} can update the correct per-type timestamp when the planner + * determines no work is needed. + */ + public void markAsPending(OptimizingType cronType) { + this.pendingCronType = cronType; + store() + .begin() + .updateStatusCode( + code -> { + if (code == OptimizingStatus.IDLE.getCode()) { + LOG.info( + "{} status changed from idle to pending (cron-triggered, type={})", + getTableIdentifier(), + cronType); + return OptimizingStatus.PENDING.getCode(); + } + return code; + }) + .commit(); + } + + /** + * Writes a {@link ProcessStatus#SKIPPED} record to the {@code table_process} table so that the + * UI can show why a cron-triggered optimization did not run. Both the insert and the subsequent + * update (which sets {@code finish_time} and {@code fail_message}) run in a single transaction so + * a partial write can never occur. + * + * @param type the optimization type whose cron fired + * @param reason human-readable skip reason + */ + public void recordSkippedOptimization(OptimizingType type, String reason) { + long now = System.currentTimeMillis(); + long processId = new SnowflakeIdGenerator().generateId(); + Map summary = new java.util.HashMap<>(); + summary.put("skipReason", reason); + summary.put("optimizingType", type.name()); + doAs( + TableProcessMapper.class, + mapper -> { + mapper.insertProcess( + getTableIdentifier().getId(), + processId, + "", + ProcessStatus.SKIPPED, + type.name().toUpperCase(), + ProcessStatus.SKIPPED.name().toLowerCase(), + "AMORO", + 0, + now, + new java.util.HashMap<>(), + summary); + mapper.updateProcess( + getTableIdentifier().getId(), + processId, + "", + ProcessStatus.SKIPPED, + ProcessStatus.SKIPPED.name().toLowerCase(), + 0, + now, + reason, + new java.util.HashMap<>(), + summary); + }); + store() + .begin() + .updateState( + OPTIMIZING_STATE_KEY, + state -> { + if (type == OptimizingType.MINOR) { + state.setLastMinorOptimizingTime(now); + } else if (type == OptimizingType.MAJOR) { + state.setLastMajorOptimizingTime(now); + } else if (type == OptimizingType.FULL) { + state.setLastFullOptimizingTime(now); + } + return state; + }) + .commit(); + LOG.info( + "[cron-skip] table={} type={} skip record persisted, reason: {}", + getTableIdentifier(), + type, + reason); + } + public void beginCommitting() { OptimizingStatus originalStatus = getOptimizingStatus(); store().begin().updateStatusCode(code -> OptimizingStatus.COMMITTING.getCode()).commit(); diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/table/TableConfigurations.java b/amoro-ams/src/main/java/org/apache/amoro/server/table/TableConfigurations.java index 34e015f489..fc9a771886 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/table/TableConfigurations.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/table/TableConfigurations.java @@ -289,26 +289,11 @@ public static OptimizingConfig parseOptimizingConfig(Map propert properties, TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_FILE_CNT, TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_FILE_CNT_DEFAULT)) - .setMinorLeastInterval( - CompatiblePropertyUtil.propertyAsInt( - properties, - TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL, - TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL_DEFAULT)) .setMajorDuplicateRatio( CompatiblePropertyUtil.propertyAsDouble( properties, TableProperties.SELF_OPTIMIZING_MAJOR_TRIGGER_DUPLICATE_RATIO, TableProperties.SELF_OPTIMIZING_MAJOR_TRIGGER_DUPLICATE_RATIO_DEFAULT)) - .setMajorTriggerInterval( - CompatiblePropertyUtil.propertyAsInt( - properties, - TableProperties.SELF_OPTIMIZING_MAJOR_TRIGGER_INTERVAL, - TableProperties.SELF_OPTIMIZING_MAJOR_TRIGGER_INTERVAL_DEFAULT)) - .setFullTriggerInterval( - CompatiblePropertyUtil.propertyAsInt( - properties, - TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL, - TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL_DEFAULT)) .setFullRewriteAllFiles( CompatiblePropertyUtil.propertyAsBoolean( properties, @@ -348,7 +333,16 @@ public static OptimizingConfig parseOptimizingConfig(Map propert PropertyUtil.propertyAsLong( properties, TableProperties.SELF_OPTIMIZING_EVALUATION_FILE_SIZE_MSE_TOLERANCE, - TableProperties.SELF_OPTIMIZING_EVALUATION_FILE_SIZE_MSE_TOLERANCE_DEFAULT)); + TableProperties.SELF_OPTIMIZING_EVALUATION_FILE_SIZE_MSE_TOLERANCE_DEFAULT)) + .setMinorTriggerCron( + CompatiblePropertyUtil.propertyAsString( + properties, TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_CRON, null)) + .setMajorTriggerCron( + CompatiblePropertyUtil.propertyAsString( + properties, TableProperties.SELF_OPTIMIZING_MAJOR_TRIGGER_CRON, null)) + .setFullTriggerCron( + CompatiblePropertyUtil.propertyAsString( + properties, TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_CRON, null)); } /** diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestMixedHiveOptimizing.java b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestMixedHiveOptimizing.java index 5555c48024..d1676c6c3d 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestMixedHiveOptimizing.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestMixedHiveOptimizing.java @@ -95,7 +95,7 @@ public void testHiveKeyedTableMajorOptimizeAndMove() throws TException, IOExcept KeyedTable table = mixedTable.asKeyedTable(); // Step1: write 1 data file into base node(0,0) updateProperties(table, TableProperties.BASE_FILE_INDEX_HASH_BUCKET, 1 + ""); - updateProperties(table, TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL, 1000 + ""); + updateProperties(table, TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_CRON, "0 0 * * *"); updateProperties(table, TableProperties.SELF_OPTIMIZING_FULL_REWRITE_ALL_FILES, false + ""); writeBase(table, rangeFromTo(1, 100, "aaa", quickDateWithZone(3))); // wait Full Optimize result diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestMixedIcebergOptimizing.java b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestMixedIcebergOptimizing.java index 2b3f8e6122..5fa7a61b20 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestMixedIcebergOptimizing.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestMixedIcebergOptimizing.java @@ -178,7 +178,7 @@ public void testPkTableMajorOptimizeLeftPosDelete() { newRecord(25, "ggg" + longString, quickDateWithZone(4)), newRecord(29, "hhh" + longString, quickDateWithZone(4)))); updateProperties(table, TableProperties.ENABLE_SELF_OPTIMIZING, "true"); - updateProperties(table, TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL, "1000"); + updateProperties(table, TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_CRON, "0 0 * * *"); TableProcessMeta optimizeHistory = checker.waitOptimizeResult(); checker.assertOptimizingProcess(optimizeHistory, OptimizingType.FULL, 5, 4); @@ -186,7 +186,7 @@ public void testPkTableMajorOptimizeLeftPosDelete() { readRecords(table), 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 21, 25, 29); updateProperties(table, TableProperties.ENABLE_SELF_OPTIMIZING, "false"); - updateProperties(table, TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL, "-1"); + updateProperties(table, TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_CRON, ""); long dataFileSize = dataFiles.get(0).fileSizeInBytes(); updateProperties( table, diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestOptimizingIntegration.java b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestOptimizingIntegration.java index f8facb1b9b..7ca3602171 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestOptimizingIntegration.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestOptimizingIntegration.java @@ -177,7 +177,7 @@ private MixedTable createMixedTable( .newTableBuilder(tableIdentifier, SCHEMA) .withPrimaryKeySpec(primaryKeySpec) .withPartitionSpec(partitionSpec) - .withProperty(TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL, "1000"); + .withProperty(TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_CRON, "0 * * * *"); return tableBuilder.create(); } @@ -190,7 +190,7 @@ private void createMixedHiveTable( .newTableBuilder(tableIdentifier, SCHEMA) .withPrimaryKeySpec(primaryKeySpec) .withPartitionSpec(partitionSpec) - .withProperty(TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL, "1000"); + .withProperty(TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_CRON, "0 * * * *"); tableBuilder.create(); } diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestOptimizingQueue.java b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestOptimizingQueue.java index 342b533aaa..3226742d45 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestOptimizingQueue.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/TestOptimizingQueue.java @@ -532,7 +532,7 @@ protected DefaultTableRuntime initTableWithFiles() { mixedTable .updateProperties() .set(TableProperties.SELF_OPTIMIZING_MIN_PLAN_INTERVAL, "10") - .set(TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL, "10") + .set(TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_CRON, "0 0 * * *") .commit(); appendData(mixedTable.asUnkeyedTable(), 1); appendData(mixedTable.asUnkeyedTable(), 2); diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/flow/TestKeyedContinuousOptimizing.java b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/flow/TestKeyedContinuousOptimizing.java index a1f3c2c20e..0a1cdfd91a 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/flow/TestKeyedContinuousOptimizing.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/flow/TestKeyedContinuousOptimizing.java @@ -19,7 +19,7 @@ package org.apache.amoro.server.optimizing.flow; import static org.apache.amoro.table.TableProperties.SELF_OPTIMIZING_FULL_REWRITE_ALL_FILES; -import static org.apache.amoro.table.TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL; +import static org.apache.amoro.table.TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_CRON; import org.apache.amoro.BasicTableTestHelper; import org.apache.amoro.TableFormat; @@ -91,7 +91,7 @@ public void run() throws Exception { int recordCountOnceWrite = 2500; // close full optimize - table.updateProperties().set(SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL, "-1").commit(); + table.updateProperties().set(SELF_OPTIMIZING_FULL_TRIGGER_CRON, "").commit(); // Need move file to hive scene table.updateProperties().set(SELF_OPTIMIZING_FULL_REWRITE_ALL_FILES, "false").commit(); @@ -168,9 +168,9 @@ public void run() throws Exception { private static void mustFullCycle(MixedTable table, RunnableWithException runnable) throws Exception { - table.updateProperties().set(SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL, "1").commit(); + table.updateProperties().set(SELF_OPTIMIZING_FULL_TRIGGER_CRON, "0 0 * * *").commit(); runnable.run(); - table.updateProperties().set(SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL, "-1").commit(); + table.updateProperties().set(SELF_OPTIMIZING_FULL_TRIGGER_CRON, "").commit(); } public interface RunnableWithException { diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/flow/TestUnKeyedContinuousOptimizing.java b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/flow/TestUnKeyedContinuousOptimizing.java index fe5e30e560..6bf696d00e 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/flow/TestUnKeyedContinuousOptimizing.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/flow/TestUnKeyedContinuousOptimizing.java @@ -19,7 +19,7 @@ package org.apache.amoro.server.optimizing.flow; import static org.apache.amoro.table.TableProperties.SELF_OPTIMIZING_FULL_REWRITE_ALL_FILES; -import static org.apache.amoro.table.TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL; +import static org.apache.amoro.table.TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_CRON; import org.apache.amoro.BasicTableTestHelper; import org.apache.amoro.TableFormat; @@ -93,7 +93,7 @@ public void run() throws Exception { int recordCountOnceWrite = 1000; // close full optimize - table.updateProperties().set(SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL, "-1").commit(); + table.updateProperties().set(SELF_OPTIMIZING_FULL_TRIGGER_CRON, "").commit(); // Need move file to hive scene UnKeyedTableDataView view = @@ -161,9 +161,9 @@ private void append(int count, UnKeyedTableDataView view) throws IOException { private static void mustFullCycle(MixedTable table, RunnableWithException runnable) throws Exception { - table.updateProperties().set(SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL, "1").commit(); + table.updateProperties().set(SELF_OPTIMIZING_FULL_TRIGGER_CRON, "0 0 * * *").commit(); runnable.run(); - table.updateProperties().set(SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL, "-1").commit(); + table.updateProperties().set(SELF_OPTIMIZING_FULL_TRIGGER_CRON, "").commit(); } public interface RunnableWithException { diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/plan/MixedTablePlanTestBase.java b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/plan/MixedTablePlanTestBase.java index 679aa46122..22720ec181 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/plan/MixedTablePlanTestBase.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/plan/MixedTablePlanTestBase.java @@ -384,7 +384,7 @@ private void setFragmentRatio(List dataFiles) { protected void closeFullOptimizingInterval() { getMixedTable() .updateProperties() - .set(TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL, "-1") + .set(TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_CRON, "") .commit(); } @@ -399,7 +399,7 @@ private StructLikeMap> partitionProperty() { protected void closeMinorOptimizingInterval() { getMixedTable() .updateProperties() - .set(TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL, "-1") + .set(TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_CRON, "") .commit(); } @@ -428,7 +428,7 @@ protected void updatePartitionProperty(StructLike partition, String key, String protected void openFullOptimizing() { getMixedTable() .updateProperties() - .set(TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL, "3600") + .set(TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_CRON, "0 0 * * *") .commit(); } diff --git a/amoro-common/src/main/java/org/apache/amoro/config/OptimizingConfig.java b/amoro-common/src/main/java/org/apache/amoro/config/OptimizingConfig.java index d5b882163a..4d7dc53133 100644 --- a/amoro-common/src/main/java/org/apache/amoro/config/OptimizingConfig.java +++ b/amoro-common/src/main/java/org/apache/amoro/config/OptimizingConfig.java @@ -66,17 +66,17 @@ public class OptimizingConfig { // self-optimizing.minor.trigger.file-count private int minorLeastFileCount; - // self-optimizing.minor.trigger.interval - private int minorLeastInterval; - // self-optimizing.major.trigger.duplicate-ratio private double majorDuplicateRatio; - // self-optimizing.major.trigger.interval - private int majorTriggerInterval; + // self-optimizing.minor.trigger.cron (null = disabled) + private String minorTriggerCron; + + // self-optimizing.major.trigger.cron (null = disabled) + private String majorTriggerCron; - // self-optimizing.full.trigger.interval - private int fullTriggerInterval; + // self-optimizing.full.trigger.cron (null = disabled) + private String fullTriggerCron; // self-optimizing.full.rewrite-all-files private boolean fullRewriteAllFiles; @@ -229,39 +229,39 @@ public OptimizingConfig setMinorLeastFileCount(int minorLeastFileCount) { return this; } - public int getMinorLeastInterval() { - return minorLeastInterval; + public double getMajorDuplicateRatio() { + return majorDuplicateRatio; } - public OptimizingConfig setMinorLeastInterval(int minorLeastInterval) { - this.minorLeastInterval = minorLeastInterval; + public OptimizingConfig setMajorDuplicateRatio(double majorDuplicateRatio) { + this.majorDuplicateRatio = majorDuplicateRatio; return this; } - public double getMajorDuplicateRatio() { - return majorDuplicateRatio; + public String getMinorTriggerCron() { + return minorTriggerCron; } - public OptimizingConfig setMajorDuplicateRatio(double majorDuplicateRatio) { - this.majorDuplicateRatio = majorDuplicateRatio; + public OptimizingConfig setMinorTriggerCron(String minorTriggerCron) { + this.minorTriggerCron = minorTriggerCron; return this; } - public int getMajorTriggerInterval() { - return majorTriggerInterval; + public String getMajorTriggerCron() { + return majorTriggerCron; } - public OptimizingConfig setMajorTriggerInterval(int majorTriggerInterval) { - this.majorTriggerInterval = majorTriggerInterval; + public OptimizingConfig setMajorTriggerCron(String majorTriggerCron) { + this.majorTriggerCron = majorTriggerCron; return this; } - public int getFullTriggerInterval() { - return fullTriggerInterval; + public String getFullTriggerCron() { + return fullTriggerCron; } - public OptimizingConfig setFullTriggerInterval(int fullTriggerInterval) { - this.fullTriggerInterval = fullTriggerInterval; + public OptimizingConfig setFullTriggerCron(String fullTriggerCron) { + this.fullTriggerCron = fullTriggerCron; return this; } @@ -353,10 +353,7 @@ public boolean equals(Object o) { && fragmentRatio == that.fragmentRatio && Double.compare(minTargetSizeRatio, that.minTargetSizeRatio) == 0 && minorLeastFileCount == that.minorLeastFileCount - && minorLeastInterval == that.minorLeastInterval && Double.compare(that.majorDuplicateRatio, majorDuplicateRatio) == 0 - && majorTriggerInterval == that.majorTriggerInterval - && fullTriggerInterval == that.fullTriggerInterval && fullRewriteAllFiles == that.fullRewriteAllFiles && Objects.equal(filter, that.filter) && baseHashBucket == that.baseHashBucket @@ -365,7 +362,10 @@ public boolean equals(Object o) { && Objects.equal(optimizerGroup, that.optimizerGroup) && Objects.equal(minPlanInterval, that.minPlanInterval) && Objects.equal(evaluationMseTolerance, that.evaluationMseTolerance) - && Objects.equal(evaluationFallbackInterval, that.evaluationFallbackInterval); + && Objects.equal(evaluationFallbackInterval, that.evaluationFallbackInterval) + && Objects.equal(minorTriggerCron, that.minorTriggerCron) + && Objects.equal(majorTriggerCron, that.majorTriggerCron) + && Objects.equal(fullTriggerCron, that.fullTriggerCron); } @Override @@ -384,10 +384,7 @@ public int hashCode() { fragmentRatio, minTargetSizeRatio, minorLeastFileCount, - minorLeastInterval, majorDuplicateRatio, - majorTriggerInterval, - fullTriggerInterval, fullRewriteAllFiles, filter, baseHashBucket, @@ -395,7 +392,10 @@ public int hashCode() { hiveRefreshInterval, minPlanInterval, evaluationMseTolerance, - evaluationFallbackInterval); + evaluationFallbackInterval, + minorTriggerCron, + majorTriggerCron, + fullTriggerCron); } @Override @@ -413,10 +413,7 @@ public String toString() { .add("openFileCost", openFileCost) .add("fragmentRatio", fragmentRatio) .add("minorLeastFileCount", minorLeastFileCount) - .add("minorLeastInterval", minorLeastInterval) .add("majorDuplicateRatio", majorDuplicateRatio) - .add("majorTriggerInterval", majorTriggerInterval) - .add("fullTriggerInterval", fullTriggerInterval) .add("fullRewriteAllFiles", fullRewriteAllFiles) .add("filter", filter) .add("baseHashBucket", baseHashBucket) @@ -424,6 +421,9 @@ public String toString() { .add("hiveRefreshInterval", hiveRefreshInterval) .add("evaluationMseTolerance", evaluationMseTolerance) .add("evaluationFallbackInterval", evaluationFallbackInterval) + .add("minorTriggerCron", minorTriggerCron) + .add("majorTriggerCron", majorTriggerCron) + .add("fullTriggerCron", fullTriggerCron) .toString(); } } diff --git a/amoro-common/src/main/java/org/apache/amoro/process/ProcessStatus.java b/amoro-common/src/main/java/org/apache/amoro/process/ProcessStatus.java index 3520a21afc..1949068020 100644 --- a/amoro-common/src/main/java/org/apache/amoro/process/ProcessStatus.java +++ b/amoro-common/src/main/java/org/apache/amoro/process/ProcessStatus.java @@ -29,7 +29,9 @@ public enum ProcessStatus { CANCELED, CLOSED, KILLED, - FAILED; + FAILED, + /** Cron fired but the optimization was determined unnecessary; recorded for UI visibility. */ + SKIPPED; public ProcessStage toStage() { return new ProcessStage(name(), ordinal()); diff --git a/amoro-common/src/main/java/org/apache/amoro/utils/CronUtils.java b/amoro-common/src/main/java/org/apache/amoro/utils/CronUtils.java new file mode 100644 index 0000000000..6206c4bce5 --- /dev/null +++ b/amoro-common/src/main/java/org/apache/amoro/utils/CronUtils.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modified by Datazip Inc. in 2026 + */ + +package org.apache.amoro.utils; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.temporal.ChronoUnit; +import java.util.BitSet; + +/** + * Lightweight utility for evaluating whether a cron expression has fired in a given time window, + * and for converting a cron expression to the equivalent repeat interval in milliseconds. + * + *

Supports standard 5-field Unix cron format: {@code minute hour day-of-month month + * day-of-week} + * + *

Examples: + * + *

    + *
  • {@code "0 0 * * *"} - daily at midnight + *
  • {@code "0 12 * * *"} - daily at noon + *
  • {@code "0 0 * * 0"} - every Sunday at midnight + *
  • {@code "0 0 1 * *"} - first day of every month at midnight + *
  • {@code "0 0/6 * * *"} - every 6 hours (step from midnight) + *
+ * + *

Day-of-week uses Unix convention: 0 = Sunday, 1 = Monday, 6 = Saturday. + */ +public final class CronUtils { + + /** Maximum look-back window when searching for a cron fire (32 days). */ + private static final long MAX_SEARCH_WINDOW_MS = 32L * 24 * 60 * 60 * 1000; + + private CronUtils() {} + + // ── public API ────────────────────────────────────────────────────────────── + + /** + * Converts a 5-field UNIX cron expression to the interval in milliseconds between two + * consecutive firings (the period of the schedule). + * + *

The interval is computed by finding the first two upcoming fire times from the current + * minute and measuring the gap. For a uniform periodic expression such as {@code "* /3 * * * *"} + * (every 3 minutes) this produces exactly 180 000 ms. For a calendar-anchored expression such as + * {@code "0 0 * * *"} (midnight daily) this produces 86 400 000 ms. + * + *

Returns {@code -1L} for a {@code null}, blank, or un-parseable expression (disabled). + */ + public static long cronToIntervalMs(String cronExpr) { + if (cronExpr == null || cronExpr.trim().isEmpty()) { + return -1L; + } + try { + CronFields fields = CronFields.parse(cronExpr); + ZoneId zone = ZoneId.systemDefault(); + LocalDateTime ref = LocalDateTime.now(zone).withSecond(0).withNano(0); + + LocalDateTime t1 = nextFireTime(fields, ref, ref.plusYears(1)); + if (t1 == null) { + return -1L; + } + LocalDateTime t2 = nextFireTime(fields, t1, t1.plusYears(1)); + if (t2 == null) { + return -1L; + } + // Cron fires are always at minute boundaries, so minutes * 60 000 is exact. + return ChronoUnit.MINUTES.between(t1, t2) * 60_000L; + } catch (Exception e) { + return -1L; + } + } + + /** + * Returns true if the cron expression fired at least once in the half-open interval + * {@code (lastFiredMs, currentTimeMs]}. + * + *

If {@code cronExpr} is {@code null} or blank, always returns {@code false}. + * + *

To avoid scanning an arbitrarily large window (e.g. when a table has never been + * compacted), the search is capped at {@value #MAX_SEARCH_WINDOW_MS} ms before + * {@code currentTimeMs}. + */ + public static boolean hasFiredSince(String cronExpr, long lastFiredMs, long currentTimeMs) { + if (cronExpr == null || cronExpr.trim().isEmpty()) { + return false; + } + if (lastFiredMs >= currentTimeMs) { + return false; + } + + CronFields fields = CronFields.parse(cronExpr); + ZoneId zone = ZoneId.systemDefault(); + + // Cap the search start to avoid iterating over years of history. + long effectiveStartMs = Math.max(lastFiredMs + 1, currentTimeMs - MAX_SEARCH_WINDOW_MS); + LocalDateTime candidate = + LocalDateTime.ofInstant(Instant.ofEpochMilli(effectiveStartMs), zone) + .truncatedTo(ChronoUnit.MINUTES); + LocalDateTime end = LocalDateTime.ofInstant(Instant.ofEpochMilli(currentTimeMs), zone); + + // After truncating to a minute boundary the candidate may fall at or before lastFiredMs + // (e.g. effectiveStartMs = 20:06:45.339 → candidate = 20:06:00, but lastFiredMs = 20:06:45.338). + // We must only count fires strictly after lastFiredMs, so advance past that point. + LocalDateTime lastFiredDt = LocalDateTime.ofInstant(Instant.ofEpochMilli(lastFiredMs), zone); + if (!candidate.isAfter(lastFiredDt)) { + candidate = lastFiredDt.truncatedTo(ChronoUnit.MINUTES).plusMinutes(1); + } + + while (!candidate.isAfter(end)) { + if (fields.matches(candidate)) { + return true; + } + candidate = candidate.plusMinutes(1); + } + return false; + } + + // ── internal helpers ──────────────────────────────────────────────────────── + + /** Returns the first fire time strictly after {@code after}, or {@code null} if none found. */ + private static LocalDateTime nextFireTime( + CronFields fields, LocalDateTime after, LocalDateTime limit) { + LocalDateTime candidate = after.plusMinutes(1).withSecond(0).withNano(0); + while (!candidate.isAfter(limit)) { + if (fields.matches(candidate)) { + return candidate; + } + candidate = candidate.plusMinutes(1); + } + return null; + } + + // ── internal cron field parser ────────────────────────────────────────────── + + static final class CronFields { + + /** Supported field ranges: minute(0-59), hour(0-23), dom(1-31), month(1-12), dow(0-6). */ + private final BitSet minutes; + private final BitSet hours; + private final BitSet daysOfMonth; + private final BitSet months; + private final BitSet daysOfWeek; + + private CronFields( + BitSet minutes, + BitSet hours, + BitSet daysOfMonth, + BitSet months, + BitSet daysOfWeek) { + this.minutes = minutes; + this.hours = hours; + this.daysOfMonth = daysOfMonth; + this.months = months; + this.daysOfWeek = daysOfWeek; + } + + boolean matches(LocalDateTime dt) { + // Java ISO: Monday=1 … Sunday=7; Unix cron: Sunday=0 … Saturday=6 + int dow = dt.getDayOfWeek().getValue() % 7; + return minutes.get(dt.getMinute()) + && hours.get(dt.getHour()) + && daysOfMonth.get(dt.getDayOfMonth()) + && months.get(dt.getMonthValue()) + && daysOfWeek.get(dow); + } + + static CronFields parse(String cronExpr) { + String[] parts = cronExpr.trim().split("\\s+"); + if (parts.length != 5) { + throw new IllegalArgumentException( + "Invalid cron expression (expected 5 fields: minute hour dom month dow): " + cronExpr); + } + return new CronFields( + parseBitSet(parts[0], 0, 59), + parseBitSet(parts[1], 0, 23), + parseBitSet(parts[2], 1, 31), + parseBitSet(parts[3], 1, 12), + parseBitSet(parts[4], 0, 6)); + } + + private static BitSet parseBitSet(String field, int min, int max) { + BitSet bits = new BitSet(max + 1); + if ("*".equals(field)) { + bits.set(min, max + 1); + return bits; + } + for (String token : field.split(",")) { + token = token.trim(); + if (token.contains("/")) { + // Step expression: "*/5" or "0/15" + String[] stepParts = token.split("/", 2); + int start = "*".equals(stepParts[0]) ? min : Integer.parseInt(stepParts[0].trim()); + int step = Integer.parseInt(stepParts[1].trim()); + for (int i = start; i <= max; i += step) { + bits.set(i); + } + } else if (token.contains("-")) { + // Range expression: "1-5" + String[] rangeParts = token.split("-", 2); + int from = Integer.parseInt(rangeParts[0].trim()); + int to = Integer.parseInt(rangeParts[1].trim()); + bits.set(from, to + 1); + } else { + bits.set(Integer.parseInt(token)); + } + } + return bits; + } + } +} diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/TableRuntimeOptimizingState.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/TableRuntimeOptimizingState.java index 521b6e02dc..6707420fdb 100644 --- a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/TableRuntimeOptimizingState.java +++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/TableRuntimeOptimizingState.java @@ -28,6 +28,12 @@ public class TableRuntimeOptimizingState { private long lastMajorOptimizingTime; private long lastFullOptimizingTime; private long lastMinorOptimizingTime; + /** + * Name of the {@code OptimizingType} that last completed successfully (FULL, MAJOR, or MINOR). + * {@code null} when no optimization has ever run. Stored as a String so that the JSON blob + * remains backward-compatible when new types are added. + */ + private String lastOptimizingType; public long getCurrentSnapshotId() { return currentSnapshotId; @@ -84,4 +90,12 @@ public long getLastMinorOptimizingTime() { public void setLastMinorOptimizingTime(long lastMinorOptimizingTime) { this.lastMinorOptimizingTime = lastMinorOptimizingTime; } + + public String getLastOptimizingType() { + return lastOptimizingType; + } + + public void setLastOptimizingType(String lastOptimizingType) { + this.lastOptimizingType = lastOptimizingType; + } } diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/plan/CommonPartitionEvaluator.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/plan/CommonPartitionEvaluator.java index b1718cefb5..dd8fb7c7de 100644 --- a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/plan/CommonPartitionEvaluator.java +++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/plan/CommonPartitionEvaluator.java @@ -26,6 +26,7 @@ import org.apache.amoro.optimizing.OptimizingType; import org.apache.amoro.optimizing.evaluation.MetadataBasedEvaluationEvent; import org.apache.amoro.shade.guava32.com.google.common.base.MoreObjects; +import org.apache.amoro.utils.CronUtils; import org.apache.amoro.shade.guava32.com.google.common.base.Preconditions; import org.apache.amoro.shade.guava32.com.google.common.collect.Sets; import org.apache.amoro.utils.TableFileUtil; @@ -116,11 +117,9 @@ public CommonPartitionEvaluator( this.lastMajorOptimizingTime = lastMajorOptimizingTime; this.lastFullOptimizingTime = lastFullOptimizingTime; this.reachMajorInterval = - config.getMajorTriggerInterval() >= 0 - && planTime - lastMajorOptimizingTime > config.getMajorTriggerInterval(); + reachTrigger(config.getMajorTriggerCron(), planTime, lastMajorOptimizingTime); this.reachFullInterval = - config.getFullTriggerInterval() >= 0 - && planTime - lastFullOptimizingTime > config.getFullTriggerInterval(); + reachTrigger(config.getFullTriggerCron(), planTime, lastFullOptimizingTime); } @Override @@ -344,11 +343,11 @@ public boolean isNecessary() { return false; } } - if (isFullOptimizing()) { - necessary = isFullNecessary(); - } else { - necessary = isMajorNecessary() || isMinorNecessary(); - } + // Priority cascade: full > major > minor. + // Each type independently evaluates its own cron + data conditions. + // If full cron fired but data conditions are not met, we still fall through to + // major and minor rather than skipping them entirely. + necessary = isFullNecessary() || isMajorNecessary() || isMinorNecessary(); LOG.debug("{} necessary = {}, {}", name(), necessary, this); } return necessary; @@ -418,14 +417,21 @@ public boolean isMinorNecessary() { } protected boolean reachMinorInterval() { - return config.getMinorLeastInterval() >= 0 - && planTime - lastMinorOptimizingTime > config.getMinorLeastInterval(); + return reachTrigger(config.getMinorTriggerCron(), planTime, lastMinorOptimizingTime); } protected boolean reachFullInterval() { return reachFullInterval; } + /** + * Returns {@code true} if the cron expression has fired at least once since the last + * optimization time. Returns {@code false} when no cron is configured (disabled by default). + */ + private static boolean reachTrigger(String cronExpr, long planTime, long lastOptimizingTime) { + return CronUtils.hasFiredSince(cronExpr, lastOptimizingTime, planTime); + } + public boolean isFullNecessary() { if (!reachFullInterval()) { return false; diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java index 76354aee7e..d12fa0d679 100644 --- a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java +++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java @@ -112,21 +112,32 @@ private TableProperties() {} "self-optimizing.minor.trigger.file-count"; public static final int SELF_OPTIMIZING_MINOR_TRIGGER_FILE_CNT_DEFAULT = 12; - public static final String SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL = - "self-optimizing.minor.trigger.interval"; - public static final int SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL_DEFAULT = 3600000; // 1 h - public static final String SELF_OPTIMIZING_MAJOR_TRIGGER_DUPLICATE_RATIO = "self-optimizing.major.trigger.duplicate-ratio"; public static final double SELF_OPTIMIZING_MAJOR_TRIGGER_DUPLICATE_RATIO_DEFAULT = 0.1; - public static final String SELF_OPTIMIZING_MAJOR_TRIGGER_INTERVAL = - "self-optimizing.major.trigger.interval"; - public static final int SELF_OPTIMIZING_MAJOR_TRIGGER_INTERVAL_DEFAULT = -1; // not trigger + /** + * Cron expression (5-field Unix format: minute hour dom month dow) that controls when minor + * compaction is triggered. Example: {@code "0 * * * *"} triggers every hour on the hour. + * Default: {@code null} (disabled). + */ + public static final String SELF_OPTIMIZING_MINOR_TRIGGER_CRON = + "self-optimizing.minor.trigger.cron"; + + /** + * Cron expression (5-field Unix format) that controls when major compaction is triggered. + * Example: {@code "0 0 * * *"} triggers daily at midnight. Default: {@code null} (disabled). + */ + public static final String SELF_OPTIMIZING_MAJOR_TRIGGER_CRON = + "self-optimizing.major.trigger.cron"; - public static final String SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL = - "self-optimizing.full.trigger.interval"; - public static final int SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL_DEFAULT = -1; // not trigger + /** + * Cron expression (5-field Unix format) that controls when full compaction is triggered. + * Example: {@code "0 0 * * 0"} triggers every Sunday at midnight. Default: {@code null} + * (disabled). + */ + public static final String SELF_OPTIMIZING_FULL_TRIGGER_CRON = + "self-optimizing.full.trigger.cron"; public static final String SELF_OPTIMIZING_FULL_REWRITE_ALL_FILES = "self-optimizing.full.rewrite-all-files"; diff --git a/amoro-format-iceberg/src/test/java/org/apache/amoro/optimizing/evaluation/TestMetadataBasedEvaluationEvent.java b/amoro-format-iceberg/src/test/java/org/apache/amoro/optimizing/evaluation/TestMetadataBasedEvaluationEvent.java index 49229d4813..98be3ae14c 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/amoro/optimizing/evaluation/TestMetadataBasedEvaluationEvent.java +++ b/amoro-format-iceberg/src/test/java/org/apache/amoro/optimizing/evaluation/TestMetadataBasedEvaluationEvent.java @@ -531,10 +531,8 @@ OptimizingConfig getDefaultOptimizingConfig() { .setMaxTaskSize(TableProperties.SELF_OPTIMIZING_MAX_TASK_SIZE_DEFAULT) .setTargetQuota(TableProperties.SELF_OPTIMIZING_QUOTA_DEFAULT) .setMinorLeastFileCount(TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_FILE_CNT_DEFAULT) - .setMinorLeastInterval(TableProperties.SELF_OPTIMIZING_MINOR_TRIGGER_INTERVAL_DEFAULT) .setMajorDuplicateRatio( TableProperties.SELF_OPTIMIZING_MAJOR_TRIGGER_DUPLICATE_RATIO_DEFAULT) - .setFullTriggerInterval(TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL_DEFAULT) .setFullRewriteAllFiles(TableProperties.SELF_OPTIMIZING_FULL_REWRITE_ALL_FILES_DEFAULT) .setFilter(TableProperties.SELF_OPTIMIZING_FILTER_DEFAULT) .setBaseHashBucket(TableProperties.BASE_FILE_INDEX_HASH_BUCKET_DEFAULT) diff --git a/amoro-format-mixed/amoro-mixed-hive/src/main/java/org/apache/amoro/hive/catalog/MixedHiveTables.java b/amoro-format-mixed/amoro-mixed-hive/src/main/java/org/apache/amoro/hive/catalog/MixedHiveTables.java index bf07b2f37e..2a853bc7a9 100644 --- a/amoro-format-mixed/amoro-mixed-hive/src/main/java/org/apache/amoro/hive/catalog/MixedHiveTables.java +++ b/amoro-format-mixed/amoro-mixed-hive/src/main/java/org/apache/amoro/hive/catalog/MixedHiveTables.java @@ -188,11 +188,6 @@ protected KeyedTable createKeyedTable( fillTableProperties(tableMeta); String hiveLocation = tableMeta.getProperties().get(HiveTableProperties.BASE_HIVE_LOCATION_ROOT); - // Default 1 day - if (!tableMeta.properties.containsKey(TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL)) { - tableMeta.putToProperties(TableProperties.SELF_OPTIMIZING_FULL_TRIGGER_INTERVAL, "86400000"); - } - AuthenticatedHadoopFileIO fileIO = AuthenticatedFileIOs.buildRecoverableHadoopFileIO( tableIdentifier, diff --git a/amoro-web/src/views/tables/components/Optimizing.vue b/amoro-web/src/views/tables/components/Optimizing.vue index ebeccc05d3..c998307a1b 100644 --- a/amoro-web/src/views/tables/components/Optimizing.vue +++ b/amoro-web/src/views/tables/components/Optimizing.vue @@ -35,6 +35,7 @@ const statusMap = { CLOSED: { title: 'CLOSED', color: '#c9cdd4' }, SUCCESS: { title: 'SUCCESS', color: '#0ad787' }, FAILED: { title: 'FAILED', color: '#f5222d' }, + SKIPPED: { title: 'SKIPPED', color: '#fa8c16' }, } const STATUS_CONFIG = shallowReactive(statusMap) @@ -292,7 +293,7 @@ onMounted(() => { /> {{ record.status }}