clp/tools/deployment/package-helm/set-up-multi-dedicated-test.sh at 4ac6a7145e9e02e83d6bce9ac00095c99cdf165b · y-scope/clp · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
#!/usr/bin/env bash

# Multi-node cluster setup with dedicated worker nodes for each worker type
# Demonstrates nodeSelector scheduling with separate node pools
# TODO: Migrate into integration test

script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"

CLP_HOME="${CLP_HOME:-/tmp/clp}"
CLUSTER_NAME="${CLUSTER_NAME:-clp-test}"
NUM_COMPRESSION_NODES="${NUM_COMPRESSION_NODES:-2}"
NUM_QUERY_NODES="${NUM_QUERY_NODES:-2}"
NUM_PRESTO_NODES="${NUM_PRESTO_NODES:-2}"
COMPRESSION_WORKER_REPLICAS="${COMPRESSION_WORKER_REPLICAS:-2}"
QUERY_WORKER_REPLICAS="${QUERY_WORKER_REPLICAS:-2}"
REDUCER_REPLICAS="${REDUCER_REPLICAS:-2}"
PRESTO_WORKER_REPLICAS="${PRESTO_WORKER_REPLICAS:-2}"

# shellcheck source=.set-up-common.sh
source "${script_dir}/.set-up-common.sh"

parse_common_args "$@"

echo "=== Multi-node setup with dedicated worker nodes ==="
echo "Cluster: ${CLUSTER_NAME}"
echo "Compression nodes: ${NUM_COMPRESSION_NODES}"
echo "Query nodes: ${NUM_QUERY_NODES}"
echo "Presto nodes: ${NUM_PRESTO_NODES}"
echo "Compression workers: ${COMPRESSION_WORKER_REPLICAS}"
echo "Query workers: ${QUERY_WORKER_REPLICAS}"
echo "Reducers: ${REDUCER_REPLICAS}"
echo "Presto workers: ${PRESTO_WORKER_REPLICAS}"
echo ""

prepare_environment "${CLUSTER_NAME}"

total_workers=$((NUM_COMPRESSION_NODES + NUM_QUERY_NODES + NUM_PRESTO_NODES))

echo "Creating kind cluster..."
generate_kind_config "${total_workers}" | kind create cluster --name "${CLUSTER_NAME}" --config=-

echo "Labeling worker nodes..."
mapfile -t worker_nodes < <(kubectl get nodes --selector='!node-role.kubernetes.io/control-plane' -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n')

# Label compression nodes
for ((i = 0; i < NUM_COMPRESSION_NODES; i++)); do
    echo "Labeling ${worker_nodes[$i]} as compression node"
    kubectl label node "${worker_nodes[$i]}" yscope.io/nodeType=compression --overwrite
done

# Label query nodes
query_end=$((NUM_COMPRESSION_NODES + NUM_QUERY_NODES))
for ((i = NUM_COMPRESSION_NODES; i < query_end; i++)); do
    echo "Labeling ${worker_nodes[$i]} as query node"
    kubectl label node "${worker_nodes[$i]}" yscope.io/nodeType=query --overwrite
done

# Label Presto nodes
for ((i = query_end; i < total_workers; i++)); do
    echo "Labeling ${worker_nodes[$i]} as presto node"
    kubectl label node "${worker_nodes[$i]}" yscope.io/nodeType=presto --overwrite
done

echo "Installing Helm chart..."
helm uninstall test --ignore-not-found
sleep 2
# Word splitting is intentional: get_image_helm_args returns multiple --set flags.
# shellcheck disable=SC2046
helm install test "${script_dir}" \
    --set "distributedDeployment=true" \
    --set "compressionWorker.replicas=${COMPRESSION_WORKER_REPLICAS}" \
    --set "compressionWorker.scheduling.nodeSelector.yscope\.io/nodeType=compression" \
    --set "queryWorker.replicas=${QUERY_WORKER_REPLICAS}" \
    --set "queryWorker.scheduling.nodeSelector.yscope\.io/nodeType=query" \
    --set "reducer.replicas=${REDUCER_REPLICAS}" \
    --set "reducer.scheduling.nodeSelector.yscope\.io/nodeType=query" \
    --set "prestoWorker.replicas=${PRESTO_WORKER_REPLICAS}" \
    --set "prestoWorker.scheduling.nodeSelector.yscope\.io/nodeType=presto" \
    $(get_image_helm_args "${CLUSTER_NAME}" "${CLP_PACKAGE_IMAGE}")

wait_for_cluster_ready