Yckms auto

.gitignore

@@ -132,3 +132,4 @@ website/components/node_modules
 *.log
 
 tools/godoctests/.bin
+/yandex/release.cfg

yandex/cleanup.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}")
+. $SCRIPT_PATH/common.sh
+. $SCRIPT_PATH/release.cfg
+
+init
+cleanup

yandex/common.sh

@@ -0,0 +1,78 @@
+init() {
+  START_DIR=$(pwd)
+  trap 'cd $START_DIR' EXIT
+
+  SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
+  cd $SCRIPT_DIR
+  SCRIPT_DIR=$(pwd)
+
+  if [[ -n $WORK_DIR ]]; then
+    mkdir -p $WORK_DIR
+    cd $WORK_DIR
+  fi
+  WORK_DIR=$(pwd)
+  YCKMS_VERSION="$BASE_VERSION+yckms"
+}
+
+go_to_work_dir() {
+  cd $WORK_DIR
+}
+
+cleanup() {
+  go_to_work_dir
+  rm -rf vault
+  rm -rf vault-kms-wrapper
+}
+
+init_vault() {
+  go_to_work_dir
+  if [[ ! -d "vault" ]]; then
+    echo "Cloning vault"
+    git clone git@github.com:yandex-cloud/vault.git
+    cd vault
+    git remote add upstream git@github.com:hashicorp/vault.git
+  else
+    echo "Vault already cloned"
+    cd vault
+    git reset --hard
+  fi
+
+  echo "Synchronizing vault with upstream"
+  git checkout main
+  git pull upstream main
+  echo "Fetching tags"
+  git fetch upstream --tags
+
+  go_to_work_dir
+}
+
+init_vault_kms_wrapper() {
+  go_to_work_dir
+  if [[ ! -d "vault-kms-wrapper" ]]; then
+    echo "Cloning vault-kms-wrapper"
+    git clone git@github.com:yandex-cloud/vault-kms-wrapper.git
+    cd vault-kms-wrapper
+  else
+    echo "Vault already cloned"
+    cd vault-kms-wrapper
+  fi
+
+  echo "Refreshing main"
+  git reset --hard
+  git checkout main
+  git pull
+  git fetch -p
+
+  go_to_work_dir
+}
+
+get_kms_wrapper_version() {
+  go_to_work_dir
+  cd vault
+  CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+  git checkout $BASE_VERSION
+  KMS_WRAPPER_VERSION=$(go list -m github.com/hashicorp/go-kms-wrapping/v2 | cut -f 2 -d " ")
+  echo "Current go-kms-wrapping version: $KMS_WRAPPER_VERSION"
+  git checkout $CURRENT_BRANCH
+  go_to_work_dir
+}

yandex/release.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+set -e
+
+SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}")
+. $SCRIPT_PATH/common.sh
+. $SCRIPT_PATH/release.cfg
+
+init
+init_vault
+get_kms_wrapper_version
+
+cd vault
+
+echo "Pushing synchronized main"
+git push origin main
+
+echo "Updating yckms"
+git checkout yckms
+git rebase origin main
+git push origin yckms
+
+if git checkout $YCKMS_VERSION; then
+  echo "Branch '$YCKMS_VERSION' already exists"
+else
+  echo "Creating branch '$YCKMS_VERSION'"
+  git checkout -b $YCKMS_VERSION $BASE_VERSION
+fi
+
+PATCH_LAST_COMMIT_MSG="YCKMS patch"
+while read -r line < <(git log $BASE_VERSION..$YCKMS_VERSION --oneline --reverse --pretty=format:"%B"); do
+  if [[ $line == "$PATCH_LAST_COMMIT_MSG" ]]; then
+    HAS_YCKMS_PATCH=true
+    break
+  fi
+done
+
+if [[ "$HAS_YCKMS_PATCH" != true ]]; then
+  echo "Applying patch from yckms branch"
+  git cherry-pick --no-commit $(git log main..yckms -1000 --oneline --reverse --pretty=format:"%h" | paste -sd' ' -)
+  # cherry-pick is more stable then merge-base
+  #git diff $(git merge-base --fork-point main yckms) yckms | git apply
+  sed -i '' 's/.*VersionMetadata.*=.*""/VersionMetadata = "yckms"/' version/version_base.go
+  go fmt version/version_base.go
+  git add version/version_base.go
+  sed -i '' "s/ARG BASE_VAULT_VERSION=.*/ARG BASE_VAULT_VERSION=$BASE_VERSION/" yandex/docker/Dockerfile
+  sed -i '' "s/BASE_VAULT_VERSION=.*/BASE_VAULT_VERSION=$BASE_VERSION/" yandex/compute/install.sh
+  git add yandex/docker/Dockerfile yandex/compute/install.sh
+
+  echo "Adding github.com/yandex-cloud/vault-kms-wrapper/v2 dependency"
+  YCKMS_WRAPPER_VERSION="$KMS_WRAPPER_VERSION-$WRAPPER_SUFFIX"
+  YCKMS_WRAPPER=github.com/yandex-cloud/vault-kms-wrapper/v2@"$YCKMS_WRAPPER_VERSION"
+
+  if ! go list -m "$YCKMS_WRAPPER"; then
+    echo >&2 "Cannot find $YCKMS_WRAPPER, possible release required!"
+    exit 1
+  fi
+  go mod edit -require="$YCKMS_WRAPPER"
+  go mod tidy
+  git add go.mod go.sum
+
+  echo "Vendoring"
+  go mod vendor
+  git add vendor
+
+  echo "Committing"
+  git commit -m "$PATCH_LAST_COMMIT_MSG"
+else
+  echo "Patch is already applied"
+fi
+
+git push -f origin $YCKMS_VERSION

yandex/release_sample.cfg

@@ -0,0 +1,6 @@
+BASE_VERSION="v1.13.2"
+WORK_DIR=./tmp
+KMS_KEY=c42rf64v82v478nnn1vt
+ENDPOINT=api.il.nebius.cloud:443
+AUTH_KEY_FILE=../auth_key_il.json
+WRAPPER_SUFFIX="yckms"

yandex/release_wrapper.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+set -e
+
+SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}")
+. $SCRIPT_PATH/common.sh
+. $SCRIPT_PATH/release.cfg
+
+init
+init_vault
+init_vault_kms_wrapper
+get_kms_wrapper_version
+
+BRANCH=release/${KMS_WRAPPER_VERSION}+${WRAPPER_SUFFIX}
+TAG=${KMS_WRAPPER_VERSION}-${WRAPPER_SUFFIX}
+
+cd vault-kms-wrapper
+
+if git ls-remote --exit-code origin $BRANCH; then
+  echo >&2 "Remote branch '$BRANCH' already exists!"
+  echo >&2 "Update suffix '$WRAPPER_SUFFIX' to release new version"
+  exit 1
+fi
+
+if git ls-remote --exit-code origin $TAG; then
+  echo >&2 "Remote tag '$TAG' already exists!"
+  echo >&2 "Update suffix '$WRAPPER_SUFFIX' to release new version"
+  exit 1
+fi
+
+echo "Getting github.com/hashicorp/go-kms-wrapping/v2@$KMS_WRAPPER_VERSION"
+go get github.com/hashicorp/go-kms-wrapping/v2@$KMS_WRAPPER_VERSION
+go mod tidy
+echo "Testing"
+go test
+git add go.mod go.sum
+
+if ! git diff --cached --quiet --exit-code; then
+  echo "Committing"
+  git commit -m "Version updated $VERSION"
+else
+  echo "Nothing to commit"
+fi
+
+if git show-ref --quiet $BRANCH; then
+  git branch -D $BRANCH
+fi
+
+echo "Creating branch $BRANCH"
+git checkout -b $BRANCH
+
+go mod vendor
+git add vendor
+
+if ! git diff --cached --quiet --exit-code; then
+  echo "Committing vendor"
+  git commit -m "Vendor"
+else
+  echo "Nothing to commit"
+fi
+
+git tag $TAG -f
+
+git push origin $BRANCH
+git push origin $TAG
+
+git checkout main
+git push origin main

yandex/test_local.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+set -e
+
+SCRIPT_PATH=$(dirname "${BASH_SOURCE[0]}")
+. $SCRIPT_PATH/common.sh
+. $SCRIPT_PATH/release.cfg
+
+init
+init_vault
+cd vault
+
+echo "Testing $YCKMS_VERSION branch"
+git checkout $YCKMS_VERSION
+
+echo "Building vault"
+make bootstrap
+make dev
+
+if [[ -n $AUTH_KEY_FILE ]]; then
+  cp $SCRIPT_DIR/$AUTH_KEY_FILE auth_key.json
+fi
+
+cat >vault.hcl <<EOF
+# See https://www.vaultproject.io/docs/configuration for more details about configuration options
+
+ui = true
+
+storage "file" {
+  path = "storage"
+}
+
+# HTTP listener (insecure)
+listener "tcp" {
+  address = "127.0.0.1:8200"
+  tls_disable = 1
+}
+
+## Auto Unseal via Yandex Key Management Service (see https://cloud.yandex.ru/docs/kms/solutions/vault-secret for more details)
+seal "yandexcloudkms" {
+  kms_key_id = "$KMS_KEY"
+  endpoint = "${ENDPOINT:-api.cloud.yandex.net:443}"
+  service_account_key_file = "auth_key.json"
+}
+EOF
+
+if [[ -d "./storage" ]]; then
+  echo "Cleaning storage"
+  rm -rf ./storage/*
+fi
+
+./bin/vault server -config vault.hcl -log-level=error -log-file=vault.log &
+PID=$!
+trap 'kill $PID' EXIT
+# waiting for server to start
+sleep 5
+
+export VAULT_ADDR=http://127.0.0.1:8200
+
+echo "Initializing vault"
+export VAULT_TOKEN=$(./bin/vault operator init | grep "Initial Root Token:" | cut -f 4 -d " ")
+echo "Vault key: $VAULT_TOKEN"
+
+echo "Enabling key value storage"
+./bin/vault secrets enable -path=secret/ kv
+
+KEY=foo
+VAL=bar
+echo "Writing key-value"
+./bin/vault write secret/my-secret $KEY=$VAL
+
+echo "Reading key-value"
+ACTUAL_VAL=$(./bin/vault read -field "$KEY" secret/my-secret)
+
+if [[ "$VAL" != "$ACTUAL_VAL" ]]; then
+  echo >&2 "Invalid key '$KEY' value! Expected $VAL, but was $ACTUAL_VAL"
+  exit 1
+fi
+echo "Local test passed successfully"