From 90367a655796585cdd0f2c9b82e3a5c8965e9403 Mon Sep 17 00:00:00 2001 From: danhuawang Date: Tue, 4 Mar 2025 20:25:04 +0800 Subject: [PATCH 1/6] add gravitino chart --- .github/workflows/chart-test.yaml | 70 ++++ dev/charts/gravitino/.helmignore | 22 ++ dev/charts/gravitino/Chart.yaml | 19 + dev/charts/gravitino/README.md | 100 ++++++ dev/charts/gravitino/resources/config/init.sh | 34 ++ .../resources/files/schema-0.8.0-mysql.sql | 329 ++++++++++++++++++ dev/charts/gravitino/templates/NOTES.txt | 1 + dev/charts/gravitino/templates/_helpers.tpl | 48 +++ .../templates/gravitino-configmap.yaml | 75 ++++ .../templates/gravitino-deployment.yaml | 97 ++++++ .../gravitino/templates/gravitino-svc.yaml | 48 +++ .../gravitino/templates/mysql-init.yaml | 13 + dev/charts/gravitino/templates/role.yaml | 11 + .../gravitino/templates/rolebinding.yaml | 15 + .../gravitino/templates/serviceaccount.yaml | 7 + .../gravitino/templates/tests/test-pod.yaml | 32 ++ dev/charts/gravitino/values.yaml | 195 +++++++++++ 17 files changed, 1116 insertions(+) create mode 100644 .github/workflows/chart-test.yaml create mode 100644 dev/charts/gravitino/.helmignore create mode 100644 dev/charts/gravitino/Chart.yaml create mode 100644 dev/charts/gravitino/README.md create mode 100644 dev/charts/gravitino/resources/config/init.sh create mode 100644 dev/charts/gravitino/resources/files/schema-0.8.0-mysql.sql create mode 100644 dev/charts/gravitino/templates/NOTES.txt create mode 100644 dev/charts/gravitino/templates/_helpers.tpl create mode 100644 dev/charts/gravitino/templates/gravitino-configmap.yaml create mode 100644 dev/charts/gravitino/templates/gravitino-deployment.yaml create mode 100644 dev/charts/gravitino/templates/gravitino-svc.yaml create mode 100644 dev/charts/gravitino/templates/mysql-init.yaml create mode 100644 dev/charts/gravitino/templates/role.yaml create mode 100644 dev/charts/gravitino/templates/rolebinding.yaml create mode 100644 dev/charts/gravitino/templates/serviceaccount.yaml create mode 100644 dev/charts/gravitino/templates/tests/test-pod.yaml create mode 100644 dev/charts/gravitino/values.yaml diff --git a/.github/workflows/chart-test.yaml b/.github/workflows/chart-test.yaml new file mode 100644 index 00000000000..88ba18bdc1e --- /dev/null +++ b/.github/workflows/chart-test.yaml @@ -0,0 +1,70 @@ +name: Test Charts + +# Controls when the workflow will run +on: + pull_request: + branches: [ "main", "branch-*" ] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + changes: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + source_changes: + - .github/workflows/** + - dev/charts/** + outputs: + source_changes: ${{ steps.filter.outputs.source_changes }} + + lint-test: + needs: changes + if: needs.changes.outputs.source_changes == 'true' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Helm + uses: azure/setup-helm@v4.2.0 + with: + version: v3.17.0 + + - uses: actions/setup-python@v5.3.0 + with: + python-version: '3.x' + check-latest: true + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.7.0 + + - name: List changed + id: list-changed + run: | + changed=$(ct list-changed --chart-dirs=dev/charts --target-branch ${{ github.event.repository.default_branch }}) + if [[ -n "$changed" ]]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Lint + if: steps.list-changed.outputs.changed == 'true' + run: | + helm repo add bitnami https://charts.bitnami.com/bitnami + ct lint --chart-dirs=dev/charts --target-branch ${{ github.event.repository.default_branch }} + + - name: Create kind cluster + if: steps.list-changed.outputs.changed == 'true' + uses: helm/kind-action@v1.12.0 + + - name: Run chart-testing (install) + if: steps.list-changed.outputs.changed == 'true' + run: ct install --chart-dirs=dev/charts --target-branch ${{ github.event.repository.default_branch }} \ No newline at end of file diff --git a/dev/charts/gravitino/.helmignore b/dev/charts/gravitino/.helmignore new file mode 100644 index 00000000000..fbe01f88f25 --- /dev/null +++ b/dev/charts/gravitino/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ \ No newline at end of file diff --git a/dev/charts/gravitino/Chart.yaml b/dev/charts/gravitino/Chart.yaml new file mode 100644 index 00000000000..0b372058b4d --- /dev/null +++ b/dev/charts/gravitino/Chart.yaml @@ -0,0 +1,19 @@ +apiVersion: v2 +appVersion: "0.8.0-incubating" +description: Gravitino is a high-performance, geo-distributed, and federated metadata lake. It manages the + metadata directly in different sources, types, and regions. It also provides users with unified metadata access + for data and AI assets. +home: https://gravitino.apache.org +dependencies: + - condition: mysql.enabled + name: mysql + repository: https://charts.bitnami.com/bitnami + version: 11.1.15 +kubeVersion: '>=1.23.0-0' +maintainers: +- email: danhua@datastrato.com + name: danhua +name: gravitino +sources: +- https://github.com/apache/gravitino +version: 0.1.0 diff --git a/dev/charts/gravitino/README.md b/dev/charts/gravitino/README.md new file mode 100644 index 00000000000..b0a076f6dc8 --- /dev/null +++ b/dev/charts/gravitino/README.md @@ -0,0 +1,100 @@ +# gravitino + +Gravitino is a high-performance, geo-distributed, and federated metadata lake. It manages the +metadata directly in +different sources, types, and regions. It also provides users with unified metadata access +for data and AI assets. + +**Homepage:** + +## Maintainers + +| Name | Email | Url | +|--------|-----------------------|-----| +| danhua | danhua@datastrato.com | | + +## Source Code + +* +* + +## Values + +| Key | Default | Description | +|---------------------|---------------------------|-----------------------------------------------------------------------| +| image.repository | `"apache/gravitino"` | | +| image.tag | `"0.8.0-incubating"` | | +| image.pullPolicy | `"IfNotPresent"` | | +| image.pullSecrets | `[] ` | Optionally specify secrets for pulling images from a private registry | +| mysql.enabled | `false` | Flag to enable MySQL as the storage backend for Gravitino | +| entity.jdbcUrl | `"jdbc:h2"` | The JDBC URL for the database | +| entity.jdbcDriver | `"org.h2.Driver"` | The JDBC driver class name | +| entity.jdbcUser | `"gravitino"` | The username for the JDBC connection | +| entity.jdbcPassword | `"gravitino"` | The password for the JDBC connection | +| env | `HADOOP_USER_NAME: hdfs ` | Environment variables to pass to the container | +| resources | `{} ` | esource requests and limits for the container | + +## Deploy gravitino to your cluster + +### Update chart dependency + +```bash +cd gravitino +helm dependency update +``` + +### Deploy with default config + +```bash +helm upgrade --install gravitino ./gravitino -n gravitino --create-namespace +``` + +### Deploy with custom config + +```bash +helm upgrade --install gravitino ./gravitino -n gravitino --create-namespace --set "key1=val1,key2=val2,..." +``` + +## Deploy gravitino and MySQL, MySQL is the gravitino storage backend + +```bash +helm upgrade --install gravitino ./gravitino -n gravitino --create-namespace --set mysql.enabled=true +``` + +To disable dynamic provisioning (The default STORAGECLASS is local-path) + +```bash +helm upgrade --install gravitino ./gravitino -n gravitino --create-namespace --set mysql.enabled=true --set global.defaultStorageClass="-" +``` + +## Deploy gravitino, use the existed MySQL as the gravitino storage backend + +```bash +helm upgrade --install gravitino ./gravitino -n gravitino --create-namespace --set entity.jdbcUrl="jdbc:mysql://database-1.***.***.rds.amazonaws.com:3306/gravitino" --set entity.jdbcDriver="com.mysql.cj.jdbc.Driver" --set entity.jdbcUser=admin --set entity.jdbcPassword=admin123 +``` + +## Others + +To init the existed MySQL, run the following command: + +```bash +mysql -h database-1.***.***.rds.amazonaws.com -P 3306 -u -p < schema-0.8.0-mysql.sql +``` + +To see the "gravitino.conf", run the following command: + +```bash +kubectl get cm gravitino -n gravitino -o json | jq -r '.data["gravitino.conf"]' +``` + +To uninstall the gravitino, run the following command: + +```bash +helm uninstall gravitino -n gravitino +``` + +To package chart + +```bash +helm package gravitino +``` \ No newline at end of file diff --git a/dev/charts/gravitino/resources/config/init.sh b/dev/charts/gravitino/resources/config/init.sh new file mode 100644 index 00000000000..9fea0ecfb10 --- /dev/null +++ b/dev/charts/gravitino/resources/config/init.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +echo "Start to download the jar package of JDBC" +wget https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar -O /root/gravitino/catalogs/jdbc-mysql/libs/mysql-connector-java-8.0.27.jar +wget https://jdbc.postgresql.org/download/postgresql-42.7.0.jar -O /root/gravitino/catalogs/jdbc-postgresql/libs/postgresql-42.7.0.jar +cp /root/gravitino/catalogs/jdbc-postgresql/libs/postgresql-42.7.0.jar /root/gravitino/catalogs/lakehouse-iceberg/libs +cp /root/gravitino/catalogs/jdbc-mysql/libs/mysql-connector-java-8.0.27.jar /root/gravitino/catalogs/lakehouse-iceberg/libs +cp /root/gravitino/catalogs/jdbc-mysql/libs/mysql-connector-java-8.0.27.jar /root/gravitino/libs +echo "Finish downloading" +cp /root/gravitino/tmp/conf/* /root/gravitino/conf +# Redirect log files to container stdout and stderr +ln -sf /dev/stdout /root/gravitino/logs/gravitino-server.log +ln -sf /dev/stderr /root/gravitino/logs/gravitino-server.out +echo "Start the Gravitino Server" +/bin/bash /root/gravitino/bin/gravitino.sh start +# tail -f /dev/null diff --git a/dev/charts/gravitino/resources/files/schema-0.8.0-mysql.sql b/dev/charts/gravitino/resources/files/schema-0.8.0-mysql.sql new file mode 100644 index 00000000000..fbf8fd9c44d --- /dev/null +++ b/dev/charts/gravitino/resources/files/schema-0.8.0-mysql.sql @@ -0,0 +1,329 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file-- +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"). You may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. +-- + +CREATE TABLE IF NOT EXISTS `metalake_meta` ( + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `metalake_name` VARCHAR(128) NOT NULL COMMENT 'metalake name', + `metalake_comment` VARCHAR(256) DEFAULT '' COMMENT 'metalake comment', + `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'metalake properties', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'metalake audit info', + `schema_version` MEDIUMTEXT NOT NULL COMMENT 'metalake schema version info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'metalake current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'metalake last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'metalake deleted at', + PRIMARY KEY (`metalake_id`), + UNIQUE KEY `uk_mn_del` (`metalake_name`, `deleted_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'metalake metadata'; + +CREATE TABLE IF NOT EXISTS `catalog_meta` ( + `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', + `catalog_name` VARCHAR(128) NOT NULL COMMENT 'catalog name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `type` VARCHAR(64) NOT NULL COMMENT 'catalog type', + `provider` VARCHAR(64) NOT NULL COMMENT 'catalog provider', + `catalog_comment` VARCHAR(256) DEFAULT '' COMMENT 'catalog comment', + `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'catalog properties', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'catalog audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'catalog current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'catalog last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'catalog deleted at', + PRIMARY KEY (`catalog_id`), + UNIQUE KEY `uk_mid_cn_del` (`metalake_id`, `catalog_name`, `deleted_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'catalog metadata'; + +CREATE TABLE IF NOT EXISTS `schema_meta` ( + `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', + `schema_name` VARCHAR(128) NOT NULL COMMENT 'schema name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', + `schema_comment` VARCHAR(256) DEFAULT '' COMMENT 'schema comment', + `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'schema properties', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'schema audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'schema current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'schema last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'schema deleted at', + PRIMARY KEY (`schema_id`), + UNIQUE KEY `uk_cid_sn_del` (`catalog_id`, `schema_name`, `deleted_at`), + KEY `idx_mid` (`metalake_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'schema metadata'; + +CREATE TABLE IF NOT EXISTS `table_meta` ( + `table_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'table id', + `table_name` VARCHAR(128) NOT NULL COMMENT 'table name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', + `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'table audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'table current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'table last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'table deleted at', + PRIMARY KEY (`table_id`), + UNIQUE KEY `uk_sid_tn_del` (`schema_id`, `table_name`, `deleted_at`), + KEY `idx_mid` (`metalake_id`), + KEY `idx_cid` (`catalog_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'table metadata'; + +CREATE TABLE IF NOT EXISTS `table_column_version_info` ( + `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', + `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', + `table_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'table id', + `table_version` INT UNSIGNED NOT NULL COMMENT 'table version', + `column_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'column id', + `column_name` VARCHAR(128) NOT NULL COMMENT 'column name', + `column_position` INT UNSIGNED NOT NULL COMMENT 'column position, starting from 0', + `column_type` TEXT NOT NULL COMMENT 'column type', + `column_comment` VARCHAR(256) DEFAULT '' COMMENT 'column comment', + `column_nullable` TINYINT(1) NOT NULL DEFAULT 1 COMMENT 'column nullable, 0 is not nullable, 1 is nullable', + `column_auto_increment` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'column auto increment, 0 is not auto increment, 1 is auto increment', + `column_default_value` TEXT DEFAULT NULL COMMENT 'column default value', + `column_op_type` TINYINT(1) NOT NULL COMMENT 'column operation type, 1 is create, 2 is update, 3 is delete', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'column deleted at', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'column audit info', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_tid_ver_cid_del` (`table_id`, `table_version`, `column_id`, `deleted_at`), + KEY `idx_mid` (`metalake_id`), + KEY `idx_cid` (`catalog_id`), + KEY `idx_sid` (`schema_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'table column version info'; + +CREATE TABLE IF NOT EXISTS `fileset_meta` ( + `fileset_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'fileset id', + `fileset_name` VARCHAR(128) NOT NULL COMMENT 'fileset name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', + `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', + `type` VARCHAR(64) NOT NULL COMMENT 'fileset type', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'fileset audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'fileset current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'fileset last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'fileset deleted at', + PRIMARY KEY (`fileset_id`), + UNIQUE KEY `uk_sid_fn_del` (`schema_id`, `fileset_name`, `deleted_at`), + KEY `idx_mid` (`metalake_id`), + KEY `idx_cid` (`catalog_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'fileset metadata'; + +CREATE TABLE IF NOT EXISTS `fileset_version_info` ( + `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', + `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', + `fileset_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'fileset id', + `version` INT UNSIGNED NOT NULL COMMENT 'fileset info version', + `fileset_comment` VARCHAR(256) DEFAULT '' COMMENT 'fileset comment', + `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'fileset properties', + `storage_location` MEDIUMTEXT NOT NULL COMMENT 'fileset storage location', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'fileset deleted at', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_fid_ver_del` (`fileset_id`, `version`, `deleted_at`), + KEY `idx_mid` (`metalake_id`), + KEY `idx_cid` (`catalog_id`), + KEY `idx_sid` (`schema_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'fileset version info'; + +CREATE TABLE IF NOT EXISTS `topic_meta` ( + `topic_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'topic id', + `topic_name` VARCHAR(128) NOT NULL COMMENT 'topic name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', + `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', + `comment` VARCHAR(256) DEFAULT '' COMMENT 'topic comment', + `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'topic properties', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'topic audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'topic current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'topic last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'topic deleted at', + PRIMARY KEY (`topic_id`), + UNIQUE KEY `uk_sid_tn_del` (`schema_id`, `topic_name`, `deleted_at`), + KEY `idx_mid` (`metalake_id`), + KEY `idx_cid` (`catalog_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'topic metadata'; + +CREATE TABLE IF NOT EXISTS `user_meta` ( + `user_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'user id', + `user_name` VARCHAR(128) NOT NULL COMMENT 'username', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'user audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'user current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'user last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'user deleted at', + PRIMARY KEY (`user_id`), + UNIQUE KEY `uk_mid_us_del` (`metalake_id`, `user_name`, `deleted_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'user metadata'; + +CREATE TABLE IF NOT EXISTS `role_meta` ( + `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', + `role_name` VARCHAR(128) NOT NULL COMMENT 'role name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'schema properties', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'role audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'role current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'role last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'role deleted at', + PRIMARY KEY (`role_id`), + UNIQUE KEY `uk_mid_rn_del` (`metalake_id`, `role_name`, `deleted_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'role metadata'; + +CREATE TABLE IF NOT EXISTS `role_meta_securable_object` ( + `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', + `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', + `metadata_object_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'The entity id of securable object', + `type` VARCHAR(128) NOT NULL COMMENT 'securable object type', + `privilege_names` TEXT(81920) NOT NULL COMMENT 'securable object privilege names', + `privilege_conditions` TEXT(81920) NOT NULL COMMENT 'securable object privilege conditions', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'securable objectcurrent version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'securable object last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'securable object deleted at', + PRIMARY KEY (`id`), + KEY `idx_obj_rid` (`role_id`), + KEY `idx_obj_eid` (`metadata_object_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'securable object meta'; + +CREATE TABLE IF NOT EXISTS `user_role_rel` ( + `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', + `user_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'user id', + `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'relation audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'relation deleted at', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_ui_ri_del` (`user_id`, `role_id`, `deleted_at`), + KEY `idx_rid` (`role_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'user role relation'; + +CREATE TABLE IF NOT EXISTS `group_meta` ( + `group_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'group id', + `group_name` VARCHAR(128) NOT NULL COMMENT 'group name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'group audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'group current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'group last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'group deleted at', + PRIMARY KEY (`group_id`), + UNIQUE KEY `uk_mid_gr_del` (`metalake_id`, `group_name`, `deleted_at`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'group metadata'; + +CREATE TABLE IF NOT EXISTS `group_role_rel` ( + `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', + `group_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'group id', + `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'relation audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'relation deleted at', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_gi_ri_del` (`group_id`, `role_id`, `deleted_at`), + KEY `idx_rid` (`group_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'group role relation'; + +CREATE TABLE IF NOT EXISTS `tag_meta` ( + `tag_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'tag id', + `tag_name` VARCHAR(128) NOT NULL COMMENT 'tag name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `tag_comment` VARCHAR(256) DEFAULT '' COMMENT 'tag comment', + `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'tag properties', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'tag audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'tag current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'tag last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'tag deleted at', + PRIMARY KEY (`tag_id`), + UNIQUE KEY `uk_mi_tn_del` (`metalake_id`, `tag_name`, `deleted_at`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'tag metadata'; + +CREATE TABLE IF NOT EXISTS `tag_relation_meta` ( + `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', + `tag_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'tag id', + `metadata_object_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metadata object id', + `metadata_object_type` VARCHAR(64) NOT NULL COMMENT 'metadata object type', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'tag relation audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'tag relation current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'tag relation last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'tag relation deleted at', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_ti_mi_mo_del` (`tag_id`, `metadata_object_id`, `metadata_object_type`, `deleted_at`), + KEY `idx_tid` (`tag_id`), + KEY `idx_mid` (`metadata_object_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'tag metadata object relation'; + +CREATE TABLE IF NOT EXISTS `owner_meta` ( + `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `owner_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'owner id', + `owner_type` VARCHAR(64) NOT NULL COMMENT 'owner type', + `metadata_object_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metadata object id', + `metadata_object_type` VARCHAR(64) NOT NULL COMMENT 'metadata object type', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'owner relation audit info', + `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'owner relation current version', + `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'owner relation last version', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'owner relation deleted at', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_ow_me_del` (`owner_id`, `metadata_object_id`, `metadata_object_type`,`deleted_at`), + KEY `idx_oid` (`owner_id`), + KEY `idx_meid` (`metadata_object_id`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'owner relation'; + +CREATE TABLE IF NOT EXISTS `model_meta` ( + `model_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'model id', + `model_name` VARCHAR(128) NOT NULL COMMENT 'model name', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', + `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', + `model_comment` TEXT DEFAULT NULL COMMENT 'model comment', + `model_properties` MEDIUMTEXT DEFAULT NULL COMMENT 'model properties', + `model_latest_version` INT UNSIGNED DEFAULT 0 COMMENT 'model latest version', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'model audit info', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'model deleted at', + PRIMARY KEY (`model_id`), + UNIQUE KEY `uk_sid_mn_del` (`schema_id`, `model_name`, `deleted_at`), + KEY `idx_mid` (`metalake_id`), + KEY `idx_cid` (`catalog_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'model metadata'; + +CREATE TABLE IF NOT EXISTS `model_version_info` ( + `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', + `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', + `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', + `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', + `model_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'model id', + `version` INT UNSIGNED NOT NULL COMMENT 'model version', + `model_version_comment` TEXT DEFAULT NULL COMMENT 'model version comment', + `model_version_properties` MEDIUMTEXT DEFAULT NULL COMMENT 'model version properties', + `model_version_uri` TEXT NOT NULL COMMENT 'model storage uri', + `audit_info` MEDIUMTEXT NOT NULL COMMENT 'model version audit info', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'model version deleted at', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_mid_ver_del` (`model_id`, `version`, `deleted_at`), + KEY `idx_mid` (`metalake_id`), + KEY `idx_cid` (`catalog_id`), + KEY `idx_sid` (`schema_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'model version info'; + +CREATE TABLE IF NOT EXISTS `model_version_alias_rel` ( + `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', + `model_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'model id', + `model_version` INT UNSIGNED NOT NULL COMMENT 'model version', + `model_version_alias` VARCHAR(128) NOT NULL COMMENT 'model version alias', + `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'model version alias deleted at', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_mi_mva_del` (`model_id`, `model_version_alias`, `deleted_at`), + KEY `idx_mva` (`model_version_alias`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'model_version_alias_rel'; diff --git a/dev/charts/gravitino/templates/NOTES.txt b/dev/charts/gravitino/templates/NOTES.txt new file mode 100644 index 00000000000..50c453fa5df --- /dev/null +++ b/dev/charts/gravitino/templates/NOTES.txt @@ -0,0 +1 @@ +The gravitino has been deployed to your cluster. \ No newline at end of file diff --git a/dev/charts/gravitino/templates/_helpers.tpl b/dev/charts/gravitino/templates/_helpers.tpl new file mode 100644 index 00000000000..91b0a62f02c --- /dev/null +++ b/dev/charts/gravitino/templates/_helpers.tpl @@ -0,0 +1,48 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "gravitino.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "gravitino.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "gravitino.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "gravitino.labels" -}} +app: {{ template "gravitino.name" . }} +chart: {{ template "gravitino.chart" . }} +release: {{ .Release.Name }} +{{- end }} + +{{/* +Define the gravitino.namespace +*/}} +{{- define "gravitino.namespace" -}} + {{- .Release.Namespace -}} +{{- end -}} diff --git a/dev/charts/gravitino/templates/gravitino-configmap.yaml b/dev/charts/gravitino/templates/gravitino-configmap.yaml new file mode 100644 index 00000000000..077bcf905e7 --- /dev/null +++ b/dev/charts/gravitino/templates/gravitino-configmap.yaml @@ -0,0 +1,75 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "gravitino.fullname" . }} + namespace: {{ include "gravitino.namespace" . }} + labels: + {{- include "gravitino.labels" . | nindent 4 }} +data: + gravitino.conf: |- + # THE CONFIGURATION FOR Gravitino SERVER + gravitino.server.shutdown.timeout = {{ .Values.server.shutdownTimeout }} + + # THE CONFIGURATION FOR Gravitino WEB SERVER + gravitino.server.webserver.host = {{ .Values.webserver.host }} + gravitino.server.webserver.httpPort = {{ .Values.webserver.httpPort }} + gravitino.server.webserver.minThreads = {{ .Values.webserver.minThreads }} + gravitino.server.webserver.maxThreads = {{ .Values.webserver.maxThreads }} + gravitino.server.webserver.stopTimeout = {{ .Values.webserver.stopTimeout }} + gravitino.server.webserver.idleTimeout = {{ .Values.webserver.idleTimeout }} + gravitino.server.webserver.threadPoolWorkQueueSize = {{ .Values.webserver.threadPoolWorkQueueSize }} + gravitino.server.webserver.requestHeaderSize = {{ .Values.webserver.requestHeaderSize }} + gravitino.server.webserver.responseHeaderSize = {{ .Values.webserver.responseHeaderSize }} + + # Comma-separated list of filter class names to apply to the API. + gravitino.server.webserver.customFilters = {{ .Values.webserver.customFilters }} + + # Comma-separated list of REST API packages to expand + gravitino.server.rest.extensionPackages = {{ .Values.server.rest.extensionPackages }} + + # THE CONFIGURATION FOR Gravitino ENTITY STORE + gravitino.entity.store = {{ .Values.entity.store }} + gravitino.entity.store.maxTransactionSkewTimeMs = {{ .Values.entity.maxTransactionSkewTimeMs }} + gravitino.entity.store.deleteAfterTimeMs = {{ (.Values.entity.deleteAfterTimeMs | default 604800000) | int }} + gravitino.entity.store.versionRetentionCount = {{ .Values.entity.versionRetentionCount }} + gravitino.entity.store.relational = {{ .Values.entity.relationalBackend }} + gravitino.entity.store.relational.jdbcUrl = {{ if .Values.mysql.enabled }}jdbc:mysql://{{ .Release.Name }}-mysql:3306/{{ .Values.mysql.auth.database }}{{ else }}{{ .Values.entity.jdbcUrl }}{{ end }} + gravitino.entity.store.relational.jdbcDriver = {{ if .Values.mysql.enabled }}com.mysql.cj.jdbc.Driver{{ else }}{{ .Values.entity.jdbcDriver }}{{ end }} + gravitino.entity.store.relational.jdbcUser = {{ if .Values.mysql.enabled }}{{ .Values.mysql.auth.username }}{{ else }}{{ .Values.entity.jdbcUser }}{{ end }} + gravitino.entity.store.relational.jdbcPassword = {{ if .Values.mysql.enabled }}{{ .Values.mysql.auth.password }}{{ else }}{{ .Values.entity.jdbcPassword }}{{ end }} + gravitino.entity.store.relational.storagePath = {{ .Values.entity.storagePath }} + + # THE CONFIGURATION FOR Gravitino CATALOG + gravitino.catalog.cache.evictionIntervalMs = {{ if .Values.catalog.evictionIntervalMs }}{{ .Values.catalog.evictionIntervalMs }}{{ else }}3600000{{ end }} + gravitino.catalog.classloader.isolated = {{ .Values.catalog.classloader.isolated }} + + # THE CONFIGURATION FOR authorization + gravitino.authorization.enable = {{ .Values.authorization.enable }} + gravitino.authorization.serviceAdmins = {{ .Values.authorization.serviceAdmins }} + gravitino.authenticators = {{ .Values.authenticators }} + gravitino.authenticator.oauth.serviceAudience = {{ .Values.authenticator.oauth.serviceAudience }} + gravitino.authenticator.oauth.defaultSignKey = {{ .Values.authenticator.oauth.defaultSignKey }} + gravitino.authenticator.oauth.serverUri = {{ .Values.authenticator.oauth.serverUri }} + gravitino.authenticator.oauth.tokenPath = {{ .Values.authenticator.oauth.tokenPath }} + + # THE CONFIGURATION FOR AUXILIARY SERVICE + gravitino.auxService.names = {{ .Values.auxService.names }} + gravitino.iceberg-rest.classpath = {{ .Values.icebergRest.classpath }} + gravitino.iceberg-rest.host = {{ .Values.icebergRest.host }} + gravitino.iceberg-rest.httpPort = {{ .Values.icebergRest.httpPort }} + gravitino.iceberg-rest.catalog-backend = {{ .Values.icebergRest.catalogBackend }} + gravitino.iceberg-rest.warehouse = {{ .Values.icebergRest.warehouse }} + + # Audit log configuration + gravitino.audit.enabled = {{ .Values.audit.enabled }} + gravitino.audit.writer.className = {{ .Values.audit.writer.className }} + gravitino.audit.formatter.className = {{ .Values.audit.formatter.className }} + gravitino.audit.writer.file.fileName = {{ .Values.audit.writer.file.fileName }} + gravitino.audit.writer.file.flushIntervalSecs = {{ .Values.audit.writer.file.flushIntervalSecs }} + gravitino.audit.writer.file.append = {{ .Values.audit.writer.file.append }} + + #Metrics configuration + gravitino.metrics.timeSlidingWindowSecs = {{ .Values.metrics.timeSlidingWindowSecs }} + + init.sh: |- + {{- tpl (.Files.Get "resources/config/init.sh") . | nindent 4 }} \ No newline at end of file diff --git a/dev/charts/gravitino/templates/gravitino-deployment.yaml b/dev/charts/gravitino/templates/gravitino-deployment.yaml new file mode 100644 index 00000000000..0357342ac31 --- /dev/null +++ b/dev/charts/gravitino/templates/gravitino-deployment.yaml @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "gravitino.fullname" . }} + namespace: {{ include "gravitino.namespace" . }} + labels: + {{- include "gravitino.labels" . | nindent 4 }} + annotations: + {{- toYaml .Values.annotations | nindent 4 }} +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + app: {{ include "gravitino.name" . }} + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ include "gravitino.name" . }} + name: {{ include "gravitino.fullname" . }} + release: {{ .Release.Name }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.image.pullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ include "gravitino.name" . }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + {{- if .Values.command }} + command: + {{- range .Values.command }} + - {{ . | quote }} + {{- end }} + {{- end }} + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + securityContext: + {{- toYaml .Values.containerSecurityContext | nindent 12 }} + env: + {{- if .Values.env }} + {{- toYaml .Values.env | nindent 12 }} + {{- end }} + volumeMounts: + - name: gravitino-conf + mountPath: /root/gravitino/tmp/conf + - name: gravitino-log + mountPath: /root/gravitino/logs + - name: storage + mountPath: {{ .Values.entity.storagePath }} + nodeSelector: + {{- toYaml .Values.nodeSelector | nindent 8 }} + affinity: + {{- toYaml .Values.affinity | nindent 8 }} + tolerations: + {{- toYaml .Values.tolerations | nindent 8 }} + volumes: + - name: gravitino-conf + configMap: + name: {{ include "gravitino.fullname" . }} + - name: gravitino-log + emptyDir: {} + {{- if not .Values.persistence.enabled }} + - name: storage + emptyDir: {} + {{- else if .Values.persistence.existingClaim }} + - name: storage + persistentVolumeClaim: + claimName: {{ .Values.persistence.existingClaim }} + {{- else }} + volumeClaimTemplates: + - metadata: + name: storage + labels: + {{- toYaml .Values.persistence.labels | nindent 8 }} + annotations: + {{- toYaml .Values.persistence.annotations | nindent 8 }} + spec: + accessModes: + {{- toYaml .Values.persistence.accessModes | nindent 8 }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + storageClassName: {{ .Values.persistence.storageClassName }} + {{- end }} \ No newline at end of file diff --git a/dev/charts/gravitino/templates/gravitino-svc.yaml b/dev/charts/gravitino/templates/gravitino-svc.yaml new file mode 100644 index 00000000000..2133c17574a --- /dev/null +++ b/dev/charts/gravitino/templates/gravitino-svc.yaml @@ -0,0 +1,48 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.service.name }} + namespace: {{ include "gravitino.namespace" . }} + labels: + {{- include "gravitino.labels" . | nindent 4 }} + {{- with .Values.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- toYaml .Values.service.annotations | nindent 4 }} +spec: + {{- if (or (eq .Values.service.type "ClusterIP") (empty .Values.service.type)) }} + type: ClusterIP + {{- if .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} + {{- else if eq .Values.service.type "LoadBalancer" }} + type: {{ .Values.service.type }} + {{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + {{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- toYaml .Values.service.loadBalancerSourceRanges | nindent 4 }} + {{- end }} + {{- else }} + type: {{ .Values.service.type }} + {{- end }} + {{- if .Values.service.externalIPs }} + externalIPs: + {{- toYaml .Values.service.externalIPs | nindent 4 }} + {{- end }} + ports: + - name: {{ .Values.service.portName }} + port: {{ .Values.service.port }} + protocol: TCP + targetPort: {{ .Values.service.targetPort }} + {{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + {{- if .Values.extraExposePorts }} + {{- tpl (toYaml .Values.extraExposePorts) . | nindent 4 }} + {{- end }} + selector: + app: {{ include "gravitino.name" . }} + release: {{ .Release.Name }} \ No newline at end of file diff --git a/dev/charts/gravitino/templates/mysql-init.yaml b/dev/charts/gravitino/templates/mysql-init.yaml new file mode 100644 index 00000000000..e8290e6cb74 --- /dev/null +++ b/dev/charts/gravitino/templates/mysql-init.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: gravitino-mysql-init + namespace: {{ include "gravitino.namespace" . }} + labels: + {{- include "gravitino.labels" . | nindent 4 }} +data: + {{- $sqlFiles := .Files.Glob "resources/files/*.sql" }} + {{- range $path, $content := $sqlFiles }} + {{ base $path }}: |- + {{- tpl ($content | toString) $ | nindent 4 }} + {{- end }} \ No newline at end of file diff --git a/dev/charts/gravitino/templates/role.yaml b/dev/charts/gravitino/templates/role.yaml new file mode 100644 index 00000000000..cf350588e79 --- /dev/null +++ b/dev/charts/gravitino/templates/role.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }}-pod-reader + namespace: {{ include "gravitino.namespace" . }} + labels: + {{- include "gravitino.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] \ No newline at end of file diff --git a/dev/charts/gravitino/templates/rolebinding.yaml b/dev/charts/gravitino/templates/rolebinding.yaml new file mode 100644 index 00000000000..ab00c3d2fdc --- /dev/null +++ b/dev/charts/gravitino/templates/rolebinding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-read-pods + namespace: {{ include "gravitino.namespace" . }} + labels: + {{- include "gravitino.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-pod-reader + namespace: {{ include "gravitino.namespace" . }} +roleRef: + kind: Role + name: {{ .Release.Name }}-pod-reader + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/dev/charts/gravitino/templates/serviceaccount.yaml b/dev/charts/gravitino/templates/serviceaccount.yaml new file mode 100644 index 00000000000..c4dd4bfd0b8 --- /dev/null +++ b/dev/charts/gravitino/templates/serviceaccount.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-pod-reader + namespace: {{ include "gravitino.namespace" . }} + labels: + {{- include "gravitino.labels" . | nindent 4 }} \ No newline at end of file diff --git a/dev/charts/gravitino/templates/tests/test-pod.yaml b/dev/charts/gravitino/templates/tests/test-pod.yaml new file mode 100644 index 00000000000..213878933f2 --- /dev/null +++ b/dev/charts/gravitino/templates/tests/test-pod.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ .Release.Name }}-test-pod" + annotations: + "helm.sh/hook": test +spec: + serviceAccountName: {{ .Release.Name }}-pod-reader + containers: + - name: test-container + image: bitnami/kubectl:latest + command: + - "sh" + - "-c" + - | + echo "Checking for running pods in namespace {{ .Release.Namespace }}..." + running_pods=$(kubectl get pods -n {{ .Release.Namespace }} --field-selector=status.phase=Running -o jsonpath='{.items[*].metadata.name}') + if [ -z "$running_pods" ]; then + echo "No running pods found in namespace {{ .Release.Namespace }}." + exit 1 + else + echo "Running pods: $running_pods" + exit 0 + fi + resources: + requests: + memory: "64Mi" + cpu: "250m" + limits: + memory: "128Mi" + cpu: "500m" + restartPolicy: Never \ No newline at end of file diff --git a/dev/charts/gravitino/values.yaml b/dev/charts/gravitino/values.yaml new file mode 100644 index 00000000000..75047367621 --- /dev/null +++ b/dev/charts/gravitino/values.yaml @@ -0,0 +1,195 @@ +image: + repository: apache/gravitino + tag: latest + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## Example: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + +# Deploy mysql on K8S cluster as gravitino storage backend. +mysql: + enabled: false + image: + pullSecrets: [] + initdbScriptsConfigMap: gravitino-mysql-init + auth: + rootPassword: "admin" + createDatabase: true + database: "gravitino" + username: "gravitino" + password: "gravitino" + +# Use H2 as the default embedded database, or set up an external database for enhanced scalability and performance. +# schema-0.8.0-mysql.sql is the data init script that should be executed in the external database manually. +entity: + store: "relational" + maxTransactionSkewTimeMs: 2000 + deleteAfterTimeMs: 604800000 + versionRetentionCount: 1 + relationalBackend: "JDBCBackend" + jdbcUrl: jdbc:h2 + jdbcDriver: org.h2.Driver + jdbcUser: gravitino + jdbcPassword: gravitino + storagePath: /root/gravitino/data + +server: + shutdownTimeout: 3000 + rest: + extensionPackages: "" + +webserver: + host: "0.0.0.0" + httpPort: 8090 + minThreads: 24 + maxThreads: 200 + stopTimeout: 30000 + idleTimeout: 30000 + threadPoolWorkQueueSize: 100 + requestHeaderSize: 131072 + responseHeaderSize: 131072 + customFilters: "" + +catalog: + cacheEvictionIntervalMs: 3600000 + classloader: + isolated: true + +authorization: + enable: false + serviceAdmins: "anonymous" + +auxService: + names: "iceberg-rest" + +icebergRest: + classpath: "iceberg-rest-server/libs, iceberg-rest-server/conf" + host: "0.0.0.0" + httpPort: 9001 + catalogBackend: "memory" + warehouse: "/tmp/" + +authenticators: simple + +authenticator: + oauth: + serviceAudience: test + defaultSignKey: MIIAIjBNBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqmmqfSy8vH9BMKZLIjPHNTClH7yyykV/J/dfNwwicoYCBQgLOLRq59nAiJ9/WOkIq7zIXQ8iZU5XmI0bRSo3Ah7G4uF6CUbpny+n4zxwTfEEJ3dVC1Aa1g3DZbf8mN76B2ykf6z4FEbTWVBp0xMZ14YBOIuOUkN4F/x8bQNYRPl0yepLBZf6uY26f2o5+yKBcpAk7K/pGoD4v+DKpt/A8bvf4fy3JpZERsMPS/veeyIqR6f+455KLjRabBpxl5xIo8iUer5O0Ci8xbQMci4ErRRemm5n1e74Kqpz5u/TcLIPdMLXJ57fLl9t7V9BtLyRTjyooJpnl/lz+kJ+VIt1kQICNPMB + serverUri: http://57.185.211.74:8080 + tokenPath: /realms/myrealm/protocol/openid-connect/token + +audit: + enabled: false + writer: + className: "org.apache.gravitino.audit.FileAuditWriter" + file: + fileName: "gravitino_audit.log" + flushIntervalSecs: 10 + append: true + formatter: + className: "org.apache.gravitino.audit.SimpleFormatter" + +metrics: + timeSlidingWindowSecs: 60 + +## ref: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ +## If you set enabled as "True", you need : +## - create a pv which above 10Gi +## - keep storageClassName same with below setting +persistence: + enabled: false + accessModes: + - ReadWriteOnce + size: 10Gi + labels: {} + annotations: {} + # existingClaim: + # storageClassName: + +## Expose the gravitino service to be accessed from outside the cluster (LoadBalancer service). +## or access it from within the cluster (ClusterIP service). Set the service type and the port to serve it. +## ref: http://kubernetes.io/docs/user-guide/services/ +## +service: + name: gravitino + type: ClusterIP + port: 8090 + targetPort: 8090 + ## Service annotations. Can be templated. + annotations: {} + labels: {} + portName: service + +extraPorts: [] +## Additional ports to the gravitino services. Useful to expose extra container ports. +# - port: 9001 +# protocol: TCP +# name: http +# targetPort: http + +## Deployment annotations +# annotations: {} + +# Deployment replicas currently only support 1 +replicas: 1 + +## Pod Annotations +# podAnnotations: {} + +## Pod Labels +# podLabels: {} + +## Define command to be executed at startup by gravitino container +command: +- "/bin/bash" +- "/root/gravitino/tmp/conf/init.sh" + +readinessProbe: + httpGet: + path: / + port: 8090 + initialDelaySeconds: 10 + timeoutSeconds: 5 + +livenessProbe: + httpGet: + path: / + port: 8090 + initialDelaySeconds: 10 + timeoutSeconds: 5 + +containerSecurityContext: + runAsNonRoot: false + runAsUser: 0 + +# Extra env variables to pass to the gravitino container +env: + - name: HADOOP_USER_NAME + value: "hdfs" + +resources: {} +# limits: +# cpu: 100m +# memory: 128Mi +# requests: +# cpu: 100m +# memory: 128Mi + +## Node labels for pod assignment +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +# +nodeSelector: {} + +## Tolerations for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Affinity for pod assignment (evaluated as template) +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} \ No newline at end of file From b52bd561dbbac45d74e8380bebb05795fd93dbfd Mon Sep 17 00:00:00 2001 From: danhuawang Date: Wed, 5 Mar 2025 14:40:57 +0800 Subject: [PATCH 2/6] change yaml --- .github/workflows/chart-test.yaml | 27 ++++-- LICENSE | 4 + dev/charts/gravitino/.helmignore | 18 ++++ dev/charts/gravitino/Chart.yaml | 29 +++++-- dev/charts/gravitino/README.md | 24 ++++-- dev/charts/gravitino/resources/config/init.sh | 1 - dev/charts/gravitino/templates/NOTES.txt | 18 ++++ dev/charts/gravitino/templates/_helpers.tpl | 19 +++++ .../templates/gravitino-configmap.yaml | 19 +++++ .../templates/gravitino-deployment.yaml | 19 +++++ .../gravitino/templates/gravitino-svc.yaml | 19 +++++ .../gravitino/templates/mysql-init.yaml | 19 +++++ dev/charts/gravitino/templates/role.yaml | 21 ++++- .../gravitino/templates/rolebinding.yaml | 25 +++++- .../gravitino/templates/serviceaccount.yaml | 21 ++++- .../gravitino/templates/tests/test-pod.yaml | 83 ++++++++++++++----- dev/charts/gravitino/values.yaml | 28 +++++-- dev/ci/chart_schema.yaml | 56 +++++++++++++ dev/ci/lintconf.yaml | 61 ++++++++++++++ 19 files changed, 458 insertions(+), 53 deletions(-) create mode 100644 dev/ci/chart_schema.yaml create mode 100644 dev/ci/lintconf.yaml diff --git a/.github/workflows/chart-test.yaml b/.github/workflows/chart-test.yaml index 88ba18bdc1e..6e48d0a29d3 100644 --- a/.github/workflows/chart-test.yaml +++ b/.github/workflows/chart-test.yaml @@ -35,17 +35,22 @@ jobs: fetch-depth: 0 - name: Set up Helm - uses: azure/setup-helm@v4.2.0 - with: - version: v3.17.0 + uses: azure/setup-helm@v4.3.0 - uses: actions/setup-python@v5.3.0 with: python-version: '3.x' check-latest: true - - name: Set up chart-testing - uses: helm/chart-testing-action@v2.7.0 + - name: Install yamale + run: pip install yamale + + - name: Install chart-testing + run: | + curl -LO https://github.com/helm/chart-testing/releases/download/v3.12.0/chart-testing_3.12.0_linux_amd64.tar.gz + tar -xzf chart-testing_3.12.0_linux_amd64.tar.gz + sudo mv ct /usr/local/bin/ + ct version - name: List changed id: list-changed @@ -59,11 +64,17 @@ jobs: if: steps.list-changed.outputs.changed == 'true' run: | helm repo add bitnami https://charts.bitnami.com/bitnami - ct lint --chart-dirs=dev/charts --target-branch ${{ github.event.repository.default_branch }} + ct lint --chart-dirs=dev/charts --target-branch ${{ github.event.repository.default_branch }} --chart-yaml-schema=dev/ci/chart_schema.yaml --lint-conf=dev/ci/lintconf.yaml - - name: Create kind cluster + - name: Set up kind cluster if: steps.list-changed.outputs.changed == 'true' - uses: helm/kind-action@v1.12.0 + uses: container-tools/kind-action@v2 + with: + version: v0.27.0 + cluster_name: helm-test-cluster + + - name: Set up kubectl + uses: azure/setup-kubectl@v4 - name: Run chart-testing (install) if: steps.list-changed.outputs.changed == 'true' diff --git a/LICENSE b/LICENSE index e8bef4c1d37..93882720b63 100644 --- a/LICENSE +++ b/LICENSE @@ -305,6 +305,10 @@ ./authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXUser.java ./authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXUserList.java + Helm chart-testing + ./dev/ci/chart_schema.yaml + ./dev/ci/lintconf.yaml + This product bundles a third-party component under the MIT License. diff --git a/dev/charts/gravitino/.helmignore b/dev/charts/gravitino/.helmignore index fbe01f88f25..504c781888e 100644 --- a/dev/charts/gravitino/.helmignore +++ b/dev/charts/gravitino/.helmignore @@ -1,3 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# # Patterns to ignore when building packages. # This supports shell glob matching, relative path matching, and # negation (prefixed with !). Only one pattern per line. diff --git a/dev/charts/gravitino/Chart.yaml b/dev/charts/gravitino/Chart.yaml index 0b372058b4d..7fafbefdc8c 100644 --- a/dev/charts/gravitino/Chart.yaml +++ b/dev/charts/gravitino/Chart.yaml @@ -1,9 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# apiVersion: v2 appVersion: "0.8.0-incubating" -description: Gravitino is a high-performance, geo-distributed, and federated metadata lake. It manages the - metadata directly in different sources, types, and regions. It also provides users with unified metadata access - for data and AI assets. +description: Gravitino is a high-performance, geo-distributed, and federated metadata lake. It manages the metadata directly in different sources, types, and regions. It also provides users with unified metadata access for data and AI assets. home: https://gravitino.apache.org +annotations: + licenses: Apache-2.0 dependencies: - condition: mysql.enabled name: mysql @@ -11,9 +29,8 @@ dependencies: version: 11.1.15 kubeVersion: '>=1.23.0-0' maintainers: -- email: danhua@datastrato.com - name: danhua + - name: Apache Gravitino name: gravitino sources: - https://github.com/apache/gravitino -version: 0.1.0 +version: 0.1.0 \ No newline at end of file diff --git a/dev/charts/gravitino/README.md b/dev/charts/gravitino/README.md index b0a076f6dc8..a23a864db0b 100644 --- a/dev/charts/gravitino/README.md +++ b/dev/charts/gravitino/README.md @@ -1,3 +1,21 @@ + # gravitino Gravitino is a high-performance, geo-distributed, and federated metadata lake. It manages the @@ -7,12 +25,6 @@ for data and AI assets. **Homepage:** -## Maintainers - -| Name | Email | Url | -|--------|-----------------------|-----| -| danhua | danhua@datastrato.com | | - ## Source Code * diff --git a/dev/charts/gravitino/resources/config/init.sh b/dev/charts/gravitino/resources/config/init.sh index 9fea0ecfb10..3f26398eddd 100644 --- a/dev/charts/gravitino/resources/config/init.sh +++ b/dev/charts/gravitino/resources/config/init.sh @@ -31,4 +31,3 @@ ln -sf /dev/stdout /root/gravitino/logs/gravitino-server.log ln -sf /dev/stderr /root/gravitino/logs/gravitino-server.out echo "Start the Gravitino Server" /bin/bash /root/gravitino/bin/gravitino.sh start -# tail -f /dev/null diff --git a/dev/charts/gravitino/templates/NOTES.txt b/dev/charts/gravitino/templates/NOTES.txt index 50c453fa5df..77dd651a533 100644 --- a/dev/charts/gravitino/templates/NOTES.txt +++ b/dev/charts/gravitino/templates/NOTES.txt @@ -1 +1,19 @@ +{{- /* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + */}} The gravitino has been deployed to your cluster. \ No newline at end of file diff --git a/dev/charts/gravitino/templates/_helpers.tpl b/dev/charts/gravitino/templates/_helpers.tpl index 91b0a62f02c..edf00f207e5 100644 --- a/dev/charts/gravitino/templates/_helpers.tpl +++ b/dev/charts/gravitino/templates/_helpers.tpl @@ -1,3 +1,22 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + {{/* vim: set filetype=mustache: */}} {{/* Expand the name of the chart. diff --git a/dev/charts/gravitino/templates/gravitino-configmap.yaml b/dev/charts/gravitino/templates/gravitino-configmap.yaml index 077bcf905e7..1a1cae99927 100644 --- a/dev/charts/gravitino/templates/gravitino-configmap.yaml +++ b/dev/charts/gravitino/templates/gravitino-configmap.yaml @@ -1,3 +1,22 @@ +{{- /* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + */}} + apiVersion: v1 kind: ConfigMap metadata: diff --git a/dev/charts/gravitino/templates/gravitino-deployment.yaml b/dev/charts/gravitino/templates/gravitino-deployment.yaml index 0357342ac31..d0909024f6d 100644 --- a/dev/charts/gravitino/templates/gravitino-deployment.yaml +++ b/dev/charts/gravitino/templates/gravitino-deployment.yaml @@ -1,3 +1,22 @@ +{{- /* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + */}} + apiVersion: apps/v1 kind: Deployment metadata: diff --git a/dev/charts/gravitino/templates/gravitino-svc.yaml b/dev/charts/gravitino/templates/gravitino-svc.yaml index 2133c17574a..3ccc7eaa6c8 100644 --- a/dev/charts/gravitino/templates/gravitino-svc.yaml +++ b/dev/charts/gravitino/templates/gravitino-svc.yaml @@ -1,3 +1,22 @@ +{{- /* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + */}} + apiVersion: v1 kind: Service metadata: diff --git a/dev/charts/gravitino/templates/mysql-init.yaml b/dev/charts/gravitino/templates/mysql-init.yaml index e8290e6cb74..9ed9172b73b 100644 --- a/dev/charts/gravitino/templates/mysql-init.yaml +++ b/dev/charts/gravitino/templates/mysql-init.yaml @@ -1,3 +1,22 @@ +{{- /* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + */}} + apiVersion: v1 kind: ConfigMap metadata: diff --git a/dev/charts/gravitino/templates/role.yaml b/dev/charts/gravitino/templates/role.yaml index cf350588e79..c97bafceb65 100644 --- a/dev/charts/gravitino/templates/role.yaml +++ b/dev/charts/gravitino/templates/role.yaml @@ -1,7 +1,26 @@ +{{- /* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + */}} + apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: {{ .Release.Name }}-pod-reader + name: {{ include "gravitino.name" . }}-pod-reader namespace: {{ include "gravitino.namespace" . }} labels: {{- include "gravitino.labels" . | nindent 4 }} diff --git a/dev/charts/gravitino/templates/rolebinding.yaml b/dev/charts/gravitino/templates/rolebinding.yaml index ab00c3d2fdc..b92926cb521 100644 --- a/dev/charts/gravitino/templates/rolebinding.yaml +++ b/dev/charts/gravitino/templates/rolebinding.yaml @@ -1,15 +1,34 @@ +{{- /* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + */}} + apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: {{ .Release.Name }}-read-pods + name: {{ include "gravitino.name" . }}-read-pods namespace: {{ include "gravitino.namespace" . }} labels: {{- include "gravitino.labels" . | nindent 4 }} subjects: - kind: ServiceAccount - name: {{ .Release.Name }}-pod-reader + name: {{ include "gravitino.name" . }}-pod-reader namespace: {{ include "gravitino.namespace" . }} roleRef: kind: Role - name: {{ .Release.Name }}-pod-reader + name: {{ include "gravitino.name" . }}-pod-reader apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/dev/charts/gravitino/templates/serviceaccount.yaml b/dev/charts/gravitino/templates/serviceaccount.yaml index c4dd4bfd0b8..d023f1c8ce6 100644 --- a/dev/charts/gravitino/templates/serviceaccount.yaml +++ b/dev/charts/gravitino/templates/serviceaccount.yaml @@ -1,7 +1,26 @@ +{{- /* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + */}} + apiVersion: v1 kind: ServiceAccount metadata: - name: {{ .Release.Name }}-pod-reader + name: {{ include "gravitino.name" . }}-pod-reader namespace: {{ include "gravitino.namespace" . }} labels: {{- include "gravitino.labels" . | nindent 4 }} \ No newline at end of file diff --git a/dev/charts/gravitino/templates/tests/test-pod.yaml b/dev/charts/gravitino/templates/tests/test-pod.yaml index 213878933f2..9f13baabcb3 100644 --- a/dev/charts/gravitino/templates/tests/test-pod.yaml +++ b/dev/charts/gravitino/templates/tests/test-pod.yaml @@ -1,32 +1,71 @@ +{{- /* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + */}} + apiVersion: v1 kind: Pod metadata: - name: "{{ .Release.Name }}-test-pod" + name: {{ include "gravitino.name" . }}-test-pod annotations: "helm.sh/hook": test spec: - serviceAccountName: {{ .Release.Name }}-pod-reader + serviceAccountName: {{ include "gravitino.name" . }}-pod-reader containers: - - name: test-container - image: bitnami/kubectl:latest - command: - - "sh" - - "-c" - - | - echo "Checking for running pods in namespace {{ .Release.Namespace }}..." - running_pods=$(kubectl get pods -n {{ .Release.Namespace }} --field-selector=status.phase=Running -o jsonpath='{.items[*].metadata.name}') - if [ -z "$running_pods" ]; then - echo "No running pods found in namespace {{ .Release.Namespace }}." - exit 1 + - name: check-pods + image: bitnami/kubectl:latest + command: + - "sh" + - "-c" + - | + # echo "Checking for running pods in namespace {{ .Release.Namespace }}..." + # running_pods=$(kubectl get pods -n {{ .Release.Namespace }} --field-selector=status.phase=Running -o jsonpath='{.items[*].metadata.name}') + # if [ -z "$running_pods" ]; then + # echo "No running pods found in namespace {{ .Release.Namespace }}." + # exit 1 + # else + # echo "Running pods: $running_pods" + # exit 0 + # fi + maxWaitTime=$((15 * 60)) + sleepInterval=10 + elapsedTime=0 + echo "Checking for all pods in namespace {{ .Release.Namespace }} 'Running'..." + while true; do + notRunningPods=$(kubectl get pods -n {{ .Release.Namespace }} -o jsonpath='{range .items[*]}{.metadata.namespace}{"\t"}{.metadata.name}{"\t"}{.status.phase}{"\n"}{end}' | grep -v "Running\|Evicted\|Completed") + if [ -z "$notRunningPods" ]; then + echo "All pods are running." + break else - echo "Running pods: $running_pods" - exit 0 + echo "Some pods are not running yet:" + echo "$notRunningPods" + sleep $sleepInterval + elapsedTime=$((elapsedTime + sleepInterval)) + if [ "$elapsedTime" -ge "$maxWaitTime" ]; then + echo "Reached maximum wait time of 15 minutes. Exiting." + exit 1 + fi fi - resources: - requests: - memory: "64Mi" - cpu: "250m" - limits: - memory: "128Mi" - cpu: "500m" + done + resources: + requests: + memory: "64Mi" + cpu: "250m" + limits: + memory: "128Mi" + cpu: "500m" restartPolicy: Never \ No newline at end of file diff --git a/dev/charts/gravitino/values.yaml b/dev/charts/gravitino/values.yaml index 75047367621..28ba952be41 100644 --- a/dev/charts/gravitino/values.yaml +++ b/dev/charts/gravitino/values.yaml @@ -1,3 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# image: repository: apache/gravitino tag: latest @@ -14,7 +32,7 @@ image: mysql: enabled: false image: - pullSecrets: [] + pullSecrets: [] initdbScriptsConfigMap: gravitino-mysql-init auth: rootPassword: "admin" @@ -35,7 +53,7 @@ entity: jdbcDriver: org.h2.Driver jdbcUser: gravitino jdbcPassword: gravitino - storagePath: /root/gravitino/data + storagePath: /root/gravitino/data server: shutdownTimeout: 3000 @@ -79,7 +97,7 @@ authenticator: oauth: serviceAudience: test defaultSignKey: MIIAIjBNBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqmmqfSy8vH9BMKZLIjPHNTClH7yyykV/J/dfNwwicoYCBQgLOLRq59nAiJ9/WOkIq7zIXQ8iZU5XmI0bRSo3Ah7G4uF6CUbpny+n4zxwTfEEJ3dVC1Aa1g3DZbf8mN76B2ykf6z4FEbTWVBp0xMZ14YBOIuOUkN4F/x8bQNYRPl0yepLBZf6uY26f2o5+yKBcpAk7K/pGoD4v+DKpt/A8bvf4fy3JpZERsMPS/veeyIqR6f+455KLjRabBpxl5xIo8iUer5O0Ci8xbQMci4ErRRemm5n1e74Kqpz5u/TcLIPdMLXJ57fLl9t7V9BtLyRTjyooJpnl/lz+kJ+VIt1kQICNPMB - serverUri: http://57.185.211.74:8080 + serverUri: http://57.185.211.74:8080 tokenPath: /realms/myrealm/protocol/openid-connect/token audit: @@ -134,7 +152,7 @@ extraPorts: [] ## Deployment annotations # annotations: {} -# Deployment replicas currently only support 1 +# Deployment replicas replicas: 1 ## Pod Annotations @@ -192,4 +210,4 @@ tolerations: [] ## Affinity for pod assignment (evaluated as template) ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity ## -affinity: {} \ No newline at end of file +affinity: {} diff --git a/dev/ci/chart_schema.yaml b/dev/ci/chart_schema.yaml new file mode 100644 index 00000000000..c690e9d0c5e --- /dev/null +++ b/dev/ci/chart_schema.yaml @@ -0,0 +1,56 @@ +{{- /* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + */}} + +name: str() +home: str(required=False) +version: str() +apiVersion: str() +appVersion: any(str(), num(), required=False) +description: str(required=False) +keywords: list(str(), required=False) +sources: list(str(), required=False) +maintainers: list(include('maintainer'), required=False) +dependencies: list(include('dependency'), required=False) +icon: str(required=False) +engine: str(required=False) +condition: str(required=False) +tags: str(required=False) +deprecated: bool(required=False) +kubeVersion: str(required=False) +annotations: map(str(), str(), required=False) +type: str(required=False) +--- +maintainer: + name: str() + email: str(required=False) + url: str(required=False) +--- +dependency: + name: str() + version: str() + repository: str(required=False) + condition: str(required=False) + tags: list(str(), required=False) + enabled: bool(required=False) + import-values: any(list(str()), list(include('import-value')), required=False) + alias: str(required=False) +--- +import-value: + child: str() + parent: str() \ No newline at end of file diff --git a/dev/ci/lintconf.yaml b/dev/ci/lintconf.yaml new file mode 100644 index 00000000000..22c1ae76abf --- /dev/null +++ b/dev/ci/lintconf.yaml @@ -0,0 +1,61 @@ +{{- /* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + */}} + +--- +rules: + braces: + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: -1 + max-spaces-inside-empty: -1 + brackets: + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: -1 + max-spaces-inside-empty: -1 + colons: + max-spaces-before: 0 + max-spaces-after: 1 + commas: + max-spaces-before: 0 + min-spaces-after: 1 + max-spaces-after: 1 + comments: + require-starting-space: true + min-spaces-from-content: 2 + document-end: disable + document-start: disable # No --- to start a file + empty-lines: + max: 2 + max-start: 0 + max-end: 0 + hyphens: + max-spaces-after: 1 + indentation: + spaces: consistent + indent-sequences: whatever # - list indentation will handle both indentation and without + check-multi-line-strings: false + key-duplicates: enable + line-length: disable # Lines can be any length + new-line-at-end-of-file: enable + new-lines: + type: unix + trailing-spaces: enable + truthy: + level: warning \ No newline at end of file From d14ba03de1aec54b8cbb9c89bf0b1633c7013ecd Mon Sep 17 00:00:00 2001 From: danhuawang Date: Wed, 5 Mar 2025 14:45:26 +0800 Subject: [PATCH 3/6] fix issue --- dev/charts/gravitino/templates/tests/test-pod.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/dev/charts/gravitino/templates/tests/test-pod.yaml b/dev/charts/gravitino/templates/tests/test-pod.yaml index 9f13baabcb3..2843f7ea24c 100644 --- a/dev/charts/gravitino/templates/tests/test-pod.yaml +++ b/dev/charts/gravitino/templates/tests/test-pod.yaml @@ -32,15 +32,6 @@ spec: - "sh" - "-c" - | - # echo "Checking for running pods in namespace {{ .Release.Namespace }}..." - # running_pods=$(kubectl get pods -n {{ .Release.Namespace }} --field-selector=status.phase=Running -o jsonpath='{.items[*].metadata.name}') - # if [ -z "$running_pods" ]; then - # echo "No running pods found in namespace {{ .Release.Namespace }}." - # exit 1 - # else - # echo "Running pods: $running_pods" - # exit 0 - # fi maxWaitTime=$((15 * 60)) sleepInterval=10 elapsedTime=0 From 28d5553424738a3478cd24a5a44d5a1f0e7dfd7a Mon Sep 17 00:00:00 2001 From: danhuawang Date: Wed, 5 Mar 2025 15:45:36 +0800 Subject: [PATCH 4/6] fix ct lint error issue --- dev/charts/gravitino/Chart.yaml | 8 ++++--- dev/ci/chart_schema.yaml | 37 ++++++++++++++++----------------- dev/ci/lintconf.yaml | 37 ++++++++++++++++----------------- 3 files changed, 41 insertions(+), 41 deletions(-) diff --git a/dev/charts/gravitino/Chart.yaml b/dev/charts/gravitino/Chart.yaml index 7fafbefdc8c..b463414c3f5 100644 --- a/dev/charts/gravitino/Chart.yaml +++ b/dev/charts/gravitino/Chart.yaml @@ -29,8 +29,10 @@ dependencies: version: 11.1.15 kubeVersion: '>=1.23.0-0' maintainers: - - name: Apache Gravitino + - name: Gravitino + email: dev@gravitino.apache.org + url: https://gravitino.apache.org name: gravitino sources: -- https://github.com/apache/gravitino -version: 0.1.0 \ No newline at end of file + - https://github.com/apache/gravitino +version: 0.1.0 diff --git a/dev/ci/chart_schema.yaml b/dev/ci/chart_schema.yaml index c690e9d0c5e..c209822a194 100644 --- a/dev/ci/chart_schema.yaml +++ b/dev/ci/chart_schema.yaml @@ -1,22 +1,21 @@ -{{- /* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - */}} - +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# name: str() home: str(required=False) version: str() diff --git a/dev/ci/lintconf.yaml b/dev/ci/lintconf.yaml index 22c1ae76abf..c5c904f3e24 100644 --- a/dev/ci/lintconf.yaml +++ b/dev/ci/lintconf.yaml @@ -1,22 +1,21 @@ -{{- /* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - */}} - +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# --- rules: braces: From bf29410a2cd12b5f4cf3deb809d5406f5d0fc743 Mon Sep 17 00:00:00 2001 From: danhuawang Date: Thu, 6 Mar 2025 15:29:39 +0800 Subject: [PATCH 5/6] use Symbolic Link for the init sql scripts --- .github/workflows/chart-test.yaml | 8 +- dev/charts/gravitino/Chart.yaml | 2 +- dev/charts/gravitino/README.md | 30 +- dev/charts/gravitino/resources/files/mysql | 1 + .../resources/files/schema-0.8.0-mysql.sql | 329 ------------------ .../gravitino/templates/mysql-init.yaml | 22 +- dev/charts/gravitino/values.yaml | 4 +- 7 files changed, 42 insertions(+), 354 deletions(-) create mode 120000 dev/charts/gravitino/resources/files/mysql delete mode 100644 dev/charts/gravitino/resources/files/schema-0.8.0-mysql.sql diff --git a/.github/workflows/chart-test.yaml b/.github/workflows/chart-test.yaml index 6e48d0a29d3..58f764d4171 100644 --- a/.github/workflows/chart-test.yaml +++ b/.github/workflows/chart-test.yaml @@ -76,6 +76,10 @@ jobs: - name: Set up kubectl uses: azure/setup-kubectl@v4 - - name: Run chart-testing (install) + - name: Run chart-testing (install default) if: steps.list-changed.outputs.changed == 'true' - run: ct install --chart-dirs=dev/charts --target-branch ${{ github.event.repository.default_branch }} \ No newline at end of file + run: ct install --chart-dirs=dev/charts --target-branch ${{ github.event.repository.default_branch }} + + - name: Run chart-testing (install mysql enable) + if: steps.list-changed.outputs.changed == 'true' + run: ct install --chart-dirs=dev/charts --helm-extra-set-args "--set=mysql.enabled=true" --target-branch ${{ github.event.repository.default_branch }} \ No newline at end of file diff --git a/dev/charts/gravitino/Chart.yaml b/dev/charts/gravitino/Chart.yaml index b463414c3f5..fcc44ddf45e 100644 --- a/dev/charts/gravitino/Chart.yaml +++ b/dev/charts/gravitino/Chart.yaml @@ -17,7 +17,7 @@ # under the License. # apiVersion: v2 -appVersion: "0.8.0-incubating" +appVersion: "0.9.0-incubating-SNAPSHOT" description: Gravitino is a high-performance, geo-distributed, and federated metadata lake. It manages the metadata directly in different sources, types, and regions. It also provides users with unified metadata access for data and AI assets. home: https://gravitino.apache.org annotations: diff --git a/dev/charts/gravitino/README.md b/dev/charts/gravitino/README.md index a23a864db0b..c16bd17c5f5 100644 --- a/dev/charts/gravitino/README.md +++ b/dev/charts/gravitino/README.md @@ -32,26 +32,26 @@ for data and AI assets. ## Values -| Key | Default | Description | -|---------------------|---------------------------|-----------------------------------------------------------------------| -| image.repository | `"apache/gravitino"` | | -| image.tag | `"0.8.0-incubating"` | | -| image.pullPolicy | `"IfNotPresent"` | | -| image.pullSecrets | `[] ` | Optionally specify secrets for pulling images from a private registry | -| mysql.enabled | `false` | Flag to enable MySQL as the storage backend for Gravitino | -| entity.jdbcUrl | `"jdbc:h2"` | The JDBC URL for the database | -| entity.jdbcDriver | `"org.h2.Driver"` | The JDBC driver class name | -| entity.jdbcUser | `"gravitino"` | The username for the JDBC connection | -| entity.jdbcPassword | `"gravitino"` | The password for the JDBC connection | -| env | `HADOOP_USER_NAME: hdfs ` | Environment variables to pass to the container | -| resources | `{} ` | esource requests and limits for the container | +| Key | Default | Description | +|---------------------|-------------------------------|-----------------------------------------------------------------------| +| image.repository | `"apache/gravitino"` | | +| image.tag | `"0.9.0-incubating-SNAPSHOT"` | | +| image.pullPolicy | `"IfNotPresent"` | | +| image.pullSecrets | `[] ` | Optionally specify secrets for pulling images from a private registry | +| mysql.enabled | `false` | Flag to enable MySQL as the storage backend for Gravitino | +| entity.jdbcUrl | `"jdbc:h2"` | The JDBC URL for the database | +| entity.jdbcDriver | `"org.h2.Driver"` | The JDBC driver class name | +| entity.jdbcUser | `"gravitino"` | The username for the JDBC connection | +| entity.jdbcPassword | `"gravitino"` | The password for the JDBC connection | +| env | `HADOOP_USER_NAME: hdfs ` | Environment variables to pass to the container | +| resources | `{} ` | esource requests and limits for the container | ## Deploy gravitino to your cluster ### Update chart dependency ```bash -cd gravitino +cd charts/gravitino helm dependency update ``` @@ -90,7 +90,7 @@ helm upgrade --install gravitino ./gravitino -n gravitino --create-namespace --s To init the existed MySQL, run the following command: ```bash -mysql -h database-1.***.***.rds.amazonaws.com -P 3306 -u -p < schema-0.8.0-mysql.sql +mysql -h database-1.***.***.rds.amazonaws.com -P 3306 -u -p < schema-0.*.0-mysql.sql ``` To see the "gravitino.conf", run the following command: diff --git a/dev/charts/gravitino/resources/files/mysql b/dev/charts/gravitino/resources/files/mysql new file mode 120000 index 00000000000..1fadad52145 --- /dev/null +++ b/dev/charts/gravitino/resources/files/mysql @@ -0,0 +1 @@ +../../../../../scripts/mysql \ No newline at end of file diff --git a/dev/charts/gravitino/resources/files/schema-0.8.0-mysql.sql b/dev/charts/gravitino/resources/files/schema-0.8.0-mysql.sql deleted file mode 100644 index fbf8fd9c44d..00000000000 --- a/dev/charts/gravitino/resources/files/schema-0.8.0-mysql.sql +++ /dev/null @@ -1,329 +0,0 @@ --- --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file-- --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"). You may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, --- software distributed under the License is distributed on an --- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY --- KIND, either express or implied. See the License for the --- specific language governing permissions and limitations --- under the License. --- - -CREATE TABLE IF NOT EXISTS `metalake_meta` ( - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `metalake_name` VARCHAR(128) NOT NULL COMMENT 'metalake name', - `metalake_comment` VARCHAR(256) DEFAULT '' COMMENT 'metalake comment', - `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'metalake properties', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'metalake audit info', - `schema_version` MEDIUMTEXT NOT NULL COMMENT 'metalake schema version info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'metalake current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'metalake last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'metalake deleted at', - PRIMARY KEY (`metalake_id`), - UNIQUE KEY `uk_mn_del` (`metalake_name`, `deleted_at`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'metalake metadata'; - -CREATE TABLE IF NOT EXISTS `catalog_meta` ( - `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', - `catalog_name` VARCHAR(128) NOT NULL COMMENT 'catalog name', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `type` VARCHAR(64) NOT NULL COMMENT 'catalog type', - `provider` VARCHAR(64) NOT NULL COMMENT 'catalog provider', - `catalog_comment` VARCHAR(256) DEFAULT '' COMMENT 'catalog comment', - `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'catalog properties', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'catalog audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'catalog current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'catalog last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'catalog deleted at', - PRIMARY KEY (`catalog_id`), - UNIQUE KEY `uk_mid_cn_del` (`metalake_id`, `catalog_name`, `deleted_at`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'catalog metadata'; - -CREATE TABLE IF NOT EXISTS `schema_meta` ( - `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', - `schema_name` VARCHAR(128) NOT NULL COMMENT 'schema name', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', - `schema_comment` VARCHAR(256) DEFAULT '' COMMENT 'schema comment', - `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'schema properties', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'schema audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'schema current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'schema last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'schema deleted at', - PRIMARY KEY (`schema_id`), - UNIQUE KEY `uk_cid_sn_del` (`catalog_id`, `schema_name`, `deleted_at`), - KEY `idx_mid` (`metalake_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'schema metadata'; - -CREATE TABLE IF NOT EXISTS `table_meta` ( - `table_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'table id', - `table_name` VARCHAR(128) NOT NULL COMMENT 'table name', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', - `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'table audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'table current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'table last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'table deleted at', - PRIMARY KEY (`table_id`), - UNIQUE KEY `uk_sid_tn_del` (`schema_id`, `table_name`, `deleted_at`), - KEY `idx_mid` (`metalake_id`), - KEY `idx_cid` (`catalog_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'table metadata'; - -CREATE TABLE IF NOT EXISTS `table_column_version_info` ( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', - `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', - `table_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'table id', - `table_version` INT UNSIGNED NOT NULL COMMENT 'table version', - `column_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'column id', - `column_name` VARCHAR(128) NOT NULL COMMENT 'column name', - `column_position` INT UNSIGNED NOT NULL COMMENT 'column position, starting from 0', - `column_type` TEXT NOT NULL COMMENT 'column type', - `column_comment` VARCHAR(256) DEFAULT '' COMMENT 'column comment', - `column_nullable` TINYINT(1) NOT NULL DEFAULT 1 COMMENT 'column nullable, 0 is not nullable, 1 is nullable', - `column_auto_increment` TINYINT(1) NOT NULL DEFAULT 0 COMMENT 'column auto increment, 0 is not auto increment, 1 is auto increment', - `column_default_value` TEXT DEFAULT NULL COMMENT 'column default value', - `column_op_type` TINYINT(1) NOT NULL COMMENT 'column operation type, 1 is create, 2 is update, 3 is delete', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'column deleted at', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'column audit info', - PRIMARY KEY (`id`), - UNIQUE KEY `uk_tid_ver_cid_del` (`table_id`, `table_version`, `column_id`, `deleted_at`), - KEY `idx_mid` (`metalake_id`), - KEY `idx_cid` (`catalog_id`), - KEY `idx_sid` (`schema_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'table column version info'; - -CREATE TABLE IF NOT EXISTS `fileset_meta` ( - `fileset_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'fileset id', - `fileset_name` VARCHAR(128) NOT NULL COMMENT 'fileset name', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', - `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', - `type` VARCHAR(64) NOT NULL COMMENT 'fileset type', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'fileset audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'fileset current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'fileset last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'fileset deleted at', - PRIMARY KEY (`fileset_id`), - UNIQUE KEY `uk_sid_fn_del` (`schema_id`, `fileset_name`, `deleted_at`), - KEY `idx_mid` (`metalake_id`), - KEY `idx_cid` (`catalog_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'fileset metadata'; - -CREATE TABLE IF NOT EXISTS `fileset_version_info` ( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', - `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', - `fileset_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'fileset id', - `version` INT UNSIGNED NOT NULL COMMENT 'fileset info version', - `fileset_comment` VARCHAR(256) DEFAULT '' COMMENT 'fileset comment', - `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'fileset properties', - `storage_location` MEDIUMTEXT NOT NULL COMMENT 'fileset storage location', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'fileset deleted at', - PRIMARY KEY (`id`), - UNIQUE KEY `uk_fid_ver_del` (`fileset_id`, `version`, `deleted_at`), - KEY `idx_mid` (`metalake_id`), - KEY `idx_cid` (`catalog_id`), - KEY `idx_sid` (`schema_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'fileset version info'; - -CREATE TABLE IF NOT EXISTS `topic_meta` ( - `topic_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'topic id', - `topic_name` VARCHAR(128) NOT NULL COMMENT 'topic name', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', - `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', - `comment` VARCHAR(256) DEFAULT '' COMMENT 'topic comment', - `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'topic properties', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'topic audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'topic current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'topic last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'topic deleted at', - PRIMARY KEY (`topic_id`), - UNIQUE KEY `uk_sid_tn_del` (`schema_id`, `topic_name`, `deleted_at`), - KEY `idx_mid` (`metalake_id`), - KEY `idx_cid` (`catalog_id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'topic metadata'; - -CREATE TABLE IF NOT EXISTS `user_meta` ( - `user_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'user id', - `user_name` VARCHAR(128) NOT NULL COMMENT 'username', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'user audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'user current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'user last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'user deleted at', - PRIMARY KEY (`user_id`), - UNIQUE KEY `uk_mid_us_del` (`metalake_id`, `user_name`, `deleted_at`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'user metadata'; - -CREATE TABLE IF NOT EXISTS `role_meta` ( - `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', - `role_name` VARCHAR(128) NOT NULL COMMENT 'role name', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'schema properties', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'role audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'role current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'role last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'role deleted at', - PRIMARY KEY (`role_id`), - UNIQUE KEY `uk_mid_rn_del` (`metalake_id`, `role_name`, `deleted_at`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'role metadata'; - -CREATE TABLE IF NOT EXISTS `role_meta_securable_object` ( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', - `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', - `metadata_object_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'The entity id of securable object', - `type` VARCHAR(128) NOT NULL COMMENT 'securable object type', - `privilege_names` TEXT(81920) NOT NULL COMMENT 'securable object privilege names', - `privilege_conditions` TEXT(81920) NOT NULL COMMENT 'securable object privilege conditions', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'securable objectcurrent version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'securable object last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'securable object deleted at', - PRIMARY KEY (`id`), - KEY `idx_obj_rid` (`role_id`), - KEY `idx_obj_eid` (`metadata_object_id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'securable object meta'; - -CREATE TABLE IF NOT EXISTS `user_role_rel` ( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', - `user_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'user id', - `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'relation audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'relation deleted at', - PRIMARY KEY (`id`), - UNIQUE KEY `uk_ui_ri_del` (`user_id`, `role_id`, `deleted_at`), - KEY `idx_rid` (`role_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'user role relation'; - -CREATE TABLE IF NOT EXISTS `group_meta` ( - `group_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'group id', - `group_name` VARCHAR(128) NOT NULL COMMENT 'group name', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'group audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'group current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'group last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'group deleted at', - PRIMARY KEY (`group_id`), - UNIQUE KEY `uk_mid_gr_del` (`metalake_id`, `group_name`, `deleted_at`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'group metadata'; - -CREATE TABLE IF NOT EXISTS `group_role_rel` ( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', - `group_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'group id', - `role_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'role id', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'relation audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'relation last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'relation deleted at', - PRIMARY KEY (`id`), - UNIQUE KEY `uk_gi_ri_del` (`group_id`, `role_id`, `deleted_at`), - KEY `idx_rid` (`group_id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'group role relation'; - -CREATE TABLE IF NOT EXISTS `tag_meta` ( - `tag_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'tag id', - `tag_name` VARCHAR(128) NOT NULL COMMENT 'tag name', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `tag_comment` VARCHAR(256) DEFAULT '' COMMENT 'tag comment', - `properties` MEDIUMTEXT DEFAULT NULL COMMENT 'tag properties', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'tag audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'tag current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'tag last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'tag deleted at', - PRIMARY KEY (`tag_id`), - UNIQUE KEY `uk_mi_tn_del` (`metalake_id`, `tag_name`, `deleted_at`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'tag metadata'; - -CREATE TABLE IF NOT EXISTS `tag_relation_meta` ( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', - `tag_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'tag id', - `metadata_object_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metadata object id', - `metadata_object_type` VARCHAR(64) NOT NULL COMMENT 'metadata object type', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'tag relation audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'tag relation current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'tag relation last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'tag relation deleted at', - PRIMARY KEY (`id`), - UNIQUE KEY `uk_ti_mi_mo_del` (`tag_id`, `metadata_object_id`, `metadata_object_type`, `deleted_at`), - KEY `idx_tid` (`tag_id`), - KEY `idx_mid` (`metadata_object_id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'tag metadata object relation'; - -CREATE TABLE IF NOT EXISTS `owner_meta` ( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `owner_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'owner id', - `owner_type` VARCHAR(64) NOT NULL COMMENT 'owner type', - `metadata_object_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metadata object id', - `metadata_object_type` VARCHAR(64) NOT NULL COMMENT 'metadata object type', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'owner relation audit info', - `current_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'owner relation current version', - `last_version` INT UNSIGNED NOT NULL DEFAULT 1 COMMENT 'owner relation last version', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'owner relation deleted at', - PRIMARY KEY (`id`), - UNIQUE KEY `uk_ow_me_del` (`owner_id`, `metadata_object_id`, `metadata_object_type`,`deleted_at`), - KEY `idx_oid` (`owner_id`), - KEY `idx_meid` (`metadata_object_id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'owner relation'; - -CREATE TABLE IF NOT EXISTS `model_meta` ( - `model_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'model id', - `model_name` VARCHAR(128) NOT NULL COMMENT 'model name', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', - `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', - `model_comment` TEXT DEFAULT NULL COMMENT 'model comment', - `model_properties` MEDIUMTEXT DEFAULT NULL COMMENT 'model properties', - `model_latest_version` INT UNSIGNED DEFAULT 0 COMMENT 'model latest version', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'model audit info', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'model deleted at', - PRIMARY KEY (`model_id`), - UNIQUE KEY `uk_sid_mn_del` (`schema_id`, `model_name`, `deleted_at`), - KEY `idx_mid` (`metalake_id`), - KEY `idx_cid` (`catalog_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'model metadata'; - -CREATE TABLE IF NOT EXISTS `model_version_info` ( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', - `metalake_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'metalake id', - `catalog_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'catalog id', - `schema_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'schema id', - `model_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'model id', - `version` INT UNSIGNED NOT NULL COMMENT 'model version', - `model_version_comment` TEXT DEFAULT NULL COMMENT 'model version comment', - `model_version_properties` MEDIUMTEXT DEFAULT NULL COMMENT 'model version properties', - `model_version_uri` TEXT NOT NULL COMMENT 'model storage uri', - `audit_info` MEDIUMTEXT NOT NULL COMMENT 'model version audit info', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'model version deleted at', - PRIMARY KEY (`id`), - UNIQUE KEY `uk_mid_ver_del` (`model_id`, `version`, `deleted_at`), - KEY `idx_mid` (`metalake_id`), - KEY `idx_cid` (`catalog_id`), - KEY `idx_sid` (`schema_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'model version info'; - -CREATE TABLE IF NOT EXISTS `model_version_alias_rel` ( - `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'auto increment id', - `model_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'model id', - `model_version` INT UNSIGNED NOT NULL COMMENT 'model version', - `model_version_alias` VARCHAR(128) NOT NULL COMMENT 'model version alias', - `deleted_at` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'model version alias deleted at', - PRIMARY KEY (`id`), - UNIQUE KEY `uk_mi_mva_del` (`model_id`, `model_version_alias`, `deleted_at`), - KEY `idx_mva` (`model_version_alias`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT 'model_version_alias_rel'; diff --git a/dev/charts/gravitino/templates/mysql-init.yaml b/dev/charts/gravitino/templates/mysql-init.yaml index 9ed9172b73b..3bdf354a90c 100644 --- a/dev/charts/gravitino/templates/mysql-init.yaml +++ b/dev/charts/gravitino/templates/mysql-init.yaml @@ -16,7 +16,7 @@ specific language governing permissions and limitations under the License. */}} - + apiVersion: v1 kind: ConfigMap metadata: @@ -25,8 +25,20 @@ metadata: labels: {{- include "gravitino.labels" . | nindent 4 }} data: - {{- $sqlFiles := .Files.Glob "resources/files/*.sql" }} + {{- $sqlFiles := .Files.Glob "resources/files/mysql/schema-*-mysql.sql" }} + {{- $maxVersionFile := "" }} + {{- $maxVersion := "0.0.0" }} {{- range $path, $content := $sqlFiles }} - {{ base $path }}: |- - {{- tpl ($content | toString) $ | nindent 4 }} - {{- end }} \ No newline at end of file + {{- $fileName := base $path }} + {{- $version := regexFind "schema-(\\d+\\.\\d+\\.\\d+)-mysql\\.sql" $fileName | replace "schema-" "" | replace "-mysql.sql" "" }} + {{- if (semverCompare ">= 0.0.0" $version) }} + {{- if (semverCompare (printf "> %s" $maxVersion) $version) }} + {{- $maxVersion = $version }} + {{- $maxVersionFile = $path }} + {{- end }} + {{- end }} + {{- end }} + {{- if $maxVersionFile }} + {{ base $maxVersionFile }}: |- + {{- tpl (.Files.Get $maxVersionFile | toString) $ | nindent 4 }} + {{- end }} diff --git a/dev/charts/gravitino/values.yaml b/dev/charts/gravitino/values.yaml index 28ba952be41..060c5c53950 100644 --- a/dev/charts/gravitino/values.yaml +++ b/dev/charts/gravitino/values.yaml @@ -18,7 +18,7 @@ # image: repository: apache/gravitino - tag: latest + tag: 0.9.0-incubating-SNAPSHOT pullPolicy: IfNotPresent ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace) ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ @@ -42,7 +42,7 @@ mysql: password: "gravitino" # Use H2 as the default embedded database, or set up an external database for enhanced scalability and performance. -# schema-0.8.0-mysql.sql is the data init script that should be executed in the external database manually. +# schema-0.*.0-mysql.sql is the data init script that should be executed in the external database manually. entity: store: "relational" maxTransactionSkewTimeMs: 2000 From c3c5a5857ca956e89bf2dc4becfb29d0a5428872 Mon Sep 17 00:00:00 2001 From: danhuawang Date: Thu, 6 Mar 2025 20:36:51 +0800 Subject: [PATCH 6/6] Change MySQL chart version to 10.2.1 (8.0.36) to adapt for ranger db --- dev/charts/gravitino/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/charts/gravitino/Chart.yaml b/dev/charts/gravitino/Chart.yaml index fcc44ddf45e..7a9c5fb4eb5 100644 --- a/dev/charts/gravitino/Chart.yaml +++ b/dev/charts/gravitino/Chart.yaml @@ -26,7 +26,7 @@ dependencies: - condition: mysql.enabled name: mysql repository: https://charts.bitnami.com/bitnami - version: 11.1.15 + version: 10.2.1 kubeVersion: '>=1.23.0-0' maintainers: - name: Gravitino