feat: Delay most of the initialization work to the start-up stage and always regenerate internal certificates (#133)

Author: CH3CHO

bin/.gitignore

@@ -0,0 +1 @@
+test*

compose/docker-compose.yml

@@ -316,6 +316,9 @@ services:
     command:
       - -config.file=/etc/loki/config/config.yaml
       - -target=all
+    depends_on:
+      prepare:
+        condition: service_completed_successfully
     networks:
       higress-net:
         aliases:

compose/scripts/init.sh

@@ -129,21 +129,6 @@ initializeApiServer() {
   mkdir -p "$VOLUMES_ROOT/api" && cd "$_"
   checkExitCode "Creating volume for API server fails with $?"
 
-  if [ ! -f ca.key ] || [ ! -f ca.crt ]; then
-    echo "  Generating CA certificate...";
-    openssl req -nodes -new -x509 -days 36500 -keyout ca.key -out ca.crt -subj "/CN=higress-root-ca/O=higress" > /dev/null 2>&1
-    checkExitCode "  Generating CA certificate for API server fails with $?";
-  else
-    echo "  CA certificate already exists.";
-  fi
-  if [ ! -f server.key ] || [ ! -f server.crt ]; then
-    echo "  Generating server certificate..."
-    openssl req -out server.csr -new -newkey rsa:$RSA_KEY_LENGTH -nodes -keyout server.key -subj "/CN=higress-api-server/O=higress" > /dev/null 2>&1 \
-      && openssl x509 -req -days 36500 -in server.csr -CA ca.crt -CAkey ca.key -set_serial 01 -sha256 -out server.crt > /dev/null 2>&1
-    checkExitCode "  Generating server certificate fails with $?";
-  else
-    echo "  Server certificate already exists.";
-  fi
   if [ ! -f nacos.key ]; then
     echo "  Generating data encryption key..."
     if [ -z "$NACOS_DATA_ENC_KEY" ]; then
@@ -152,46 +137,7 @@ initializeApiServer() {
       echo -n "$NACOS_DATA_ENC_KEY" > nacos.key
     fi
   else
-    echo "  Client certificate already exists.";
-  fi
-  if [ ! -f client.key ] || [ ! -f client.crt ]; then
-    echo "  Generating client certificate..."
-    openssl req -out client.csr -new -newkey rsa:$RSA_KEY_LENGTH -nodes -keyout client.key -subj "/CN=higress/O=system:masters" > /dev/null 2>&1 \
-      && openssl x509 -req -days 36500 -in client.csr -CA ca.crt -CAkey ca.key -set_serial 02 -sha256 -out client.crt > /dev/null 2>&1
-    checkExitCode "  Generating client certificate fails with $?";
-  else
-    echo "  Client certificate already exists.";
-  fi
-
-  CLIENT_CERT=$(cat client.crt | base64 -w 0)
-  CLIENT_KEY=$(cat client.key | base64 -w 0)
-
-  if [ ! -f $VOLUMES_ROOT/kube/config ]; then
-    echo "  Generating kubeconfig..."
-    mkdir -p $VOLUMES_ROOT/kube
-    cat <<EOF > $VOLUMES_ROOT/kube/config
-apiVersion: v1
-kind: Config
-clusters:
-  - name: higress
-    cluster:
-      server: https://apiserver:8443
-      insecure-skip-tls-verify: true
-users:
-  - name: higress-admin
-    user:
-      client-certificate-data: ${CLIENT_CERT}
-      client-key-data: ${CLIENT_KEY}
-contexts:
-  - name: higress
-    context:
-      cluster: higress
-      user: higress-admin
-preferences: {}
-current-context: higress
-EOF
-  else
-    echo "  kubeconfig already exists."
+    echo "  Data encryption key already exists.";
   fi
 }
 
@@ -206,301 +152,7 @@ initializeController() {
   fi
 }
 
-initializePilot() {
-  echo "Initializing pilot configurations..."
-
-  mkdir -p $VOLUMES_ROOT/pilot/cacerts && cd "$_"
-
-  if [ ! -f root-key.pem ] || [ ! -f root-cert.pem ]; then
-    openssl req -newkey rsa:$RSA_KEY_LENGTH -nodes -keyout root-key.pem -x509 -days 36500 -out root-cert.pem > /dev/null 2>&1 <<EOF
-CN
-Shanghai
-Shanghai
-Higress
-Gateway
-Root CA
-[email protected]
-
-
-EOF
-    checkExitCode "  Generating Root CA certificate for pilot fails with $?"
-  fi
-
-  if [ ! -f ca-key.pem ] || [ ! -f ca-cert.pem ]; then
-    cat <<EOF > ca.cfg
-[req]
-distinguished_name = req_distinguished_name
-req_extensions = v3_req
-prompt = no
-
-[req_distinguished_name]
-C = CN
-ST = Shanghai
-L = Shanghai
-O = Higress
-CN = Higress CA
-
-[v3_req]
-keyUsage = keyCertSign
-basicConstraints = CA:TRUE
-subjectAltName = @alt_names
-
-[alt_names]
-DNS.1 = ca.higress.io
-EOF
-    openssl genrsa -out ca-key.pem $RSA_KEY_LENGTH > /dev/null \
-      && openssl req -new -key ca-key.pem -out ca-cert.csr -config ca.cfg -batch -sha256 > /dev/null 2>&1 \
-      && openssl x509 -req -days 36500 -in ca-cert.csr -sha256 -CA root-cert.pem -CAkey root-key.pem -CAcreateserial -out ca-cert.pem -extensions v3_req -extfile ca.cfg > /dev/null 2>&1
-    checkExitCode "Generating intermedia CA certificate for pilot fails with $?"
-    cp ca-cert.pem cert-chain.pem > /dev/null
-    chmod a+r ca-key.pem
-    rm ./*csr > /dev/null
-  fi
-
-  if [ ! -f gateway-key.pem ] || [ ! -f gateway-cert.pem ]; then
-    cat <<EOF > gateway.cfg
-[req]
-distinguished_name = req_distinguished_name
-req_extensions = v3_req
-prompt = no
-
-[req_distinguished_name]
-C = CN
-ST = Shanghai
-L = Shanghai
-O = Higress
-CN = Higress Gateway
-
-[v3_req]
-keyUsage = digitalSignature, keyEncipherment
-subjectAltName = URI:spiffe://cluster.local/ns/higress-system/sa/higress-gateway
-EOF
-    openssl genrsa -out gateway-key.pem $RSA_KEY_LENGTH > /dev/null \
-      && openssl req -new -key gateway-key.pem -out gateway-cert.csr -config gateway.cfg -batch -sha256 > /dev/null 2>&1 \
-      && openssl x509 -req -days 36500 -in gateway-cert.csr -sha256 -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out gateway-cert.pem -extensions v3_req -extfile gateway.cfg > /dev/null 2>&1
-    checkExitCode "Generating certificate for gateway fails with $?"
-    chmod a+r gateway-key.pem
-  fi
-}
-
-initializeGateway() {
-  echo "Initializing gateway configurations..."
-
-  mkdir -p $VOLUMES_ROOT/gateway/certs && cd "$_"
-  cp $VOLUMES_ROOT/pilot/cacerts/root-cert.pem ./root-cert.pem
-  cp $VOLUMES_ROOT/pilot/cacerts/gateway-cert.pem ./cert-chain.pem
-  cp $VOLUMES_ROOT/pilot/cacerts/gateway-key.pem ./key.pem
-  cat $VOLUMES_ROOT/pilot/cacerts/ca-cert.pem >>./cert-chain.pem
-
-  mkdir -p $VOLUMES_ROOT/gateway/podinfo && cd "$_"
-  cat <<EOF >./labels
-app="higress-gateway"
-higress="higress-system-higress-gateway"
-EOF
-
-  mkdir -p $VOLUMES_ROOT/gateway/istio/data
-
-  mkdir -p $VOLUMES_ROOT/gateway/log
-  touch $VOLUMES_ROOT/gateway/log/access.log
-}
-
-initializePrometheus() {
-  echo "Initializing Prometheus configurations..."
-
-  mkdir -p $VOLUMES_ROOT/prometheus/config && cd "$_"
-  cat <<EOF >./prometheus.yaml
-global:
-  scrape_interval:     15s 
-  evaluation_interval: 15s
-scrape_configs:
-  - job_name: 'prometheus'
-    metrics_path: /prometheus/metrics
-    static_configs:
-    - targets: ['localhost:9090']
-  - job_name: 'gateway'
-    metrics_path: /stats/prometheus
-    static_configs:
-    - targets: ['gateway:15020']
-      labels:
-        container: 'higress-gateway'
-        namespace: 'higress-system'
-        higress: 'higress-system-higress-gateway'
-        pod: 'higress'
-EOF
-
-  mkdir -p $VOLUMES_ROOT/prometheus/data
-  chmod a+rwx $VOLUMES_ROOT/prometheus/data
-}
-
-initializePromtail() {
-  echo "Initializing Promtail configurations..."
-
-  mkdir -p $VOLUMES_ROOT/promtail/config && cd "$_"
-  cat <<EOF >./promtail.yaml
-server:
-  log_level: info
-  http_listen_port: 3101
-
-clients:
-- url: http://loki:3100/loki/api/v1/push
-
-positions:
-  filename: /var/promtail/promtail-positions.yaml
-target_config:
-  sync_period: 10s
-scrape_configs:
-- job_name: access-logs
-  static_configs:
-  - targets:
-    - localhost
-    labels:
-      __path__: /var/log/proxy/access.log
-  pipeline_stages:
-  - json:
-      expressions:
-        authority:
-        method:
-        path:
-        protocol:
-        request_id:
-        response_code:
-        response_flags:
-        route_name:
-        trace_id:
-        upstream_cluster:
-        upstream_host:
-        upstream_transport_failure_reason:
-        user_agent:
-        x_forwarded_for:
-  - labels:
-      authority:
-      method:
-      path:
-      protocol:
-      request_id:
-      response_code:
-      response_flags:
-      route_name:
-      trace_id:
-      upstream_cluster:
-      upstream_host:
-      upstream_transport_failure_reason:
-      user_agent:
-      x_forwarded_for:
-  - timestamp:
-      source: timestamp
-      format: RFC3339Nano
-EOF
-  
-  mkdir -p $VOLUMES_ROOT/promtail/data
-  chmod a+rwx $VOLUMES_ROOT/promtail/data
-}
-
-initializeLoki() {
-  echo "Initializing Loki configurations..."
-
-  mkdir -p $VOLUMES_ROOT/loki/config && cd "$_"
-  cat <<EOF >./config.yaml
-auth_enabled: false
-common:
-  compactor_address: 'loki'
-  path_prefix: /var/loki
-  replication_factor: 1
-  storage:
-    filesystem:
-      chunks_directory: /var/loki/chunks
-      rules_directory: /var/loki/rules
-frontend:
-  scheduler_address: ""
-frontend_worker:
-  scheduler_address: ""
-index_gateway:
-  mode: ring
-limits_config:
-  max_cache_freshness_per_query: 10m
-  reject_old_samples: true
-  reject_old_samples_max_age: 168h
-  split_queries_by_interval: 15m
-memberlist:
-  join_members:
-  - loki
-query_range:
-  align_queries_with_step: true
-ruler:
-  storage:
-    type: local
-runtime_config:
-  file: /etc/loki/config/runtime-config.yaml
-schema_config:
-  configs:
-  - from: "2022-01-11"
-    index:
-      period: 24h
-      prefix: loki_index_
-    object_store: filesystem
-    schema: v12
-    store: boltdb-shipper
-server:
-  http_listen_port: 3100
-  grpc_listen_port: 9095
-storage_config:
-  hedging:
-    at: 250ms
-    max_per_second: 20
-    up_to: 3
-tracing:
-  enabled: false
-EOF
-  cat <<EOF >./runtime-config.yaml
-{}
-EOF
-
-  mkdir -p $VOLUMES_ROOT/loki/data/
-  chmod a+rwx $VOLUMES_ROOT/loki/data/
-}
-
-initializeGrafana() {
-  echo "Initializing Grafana configurations..."
-
-  mkdir -p $VOLUMES_ROOT/grafana/config && cd "$_"
-  cat <<EOF >./grafana.ini
-[server]
-protocol=http
-domain=localhost
-root_url="%(protocol)s://%(domain)s/grafana"
-serve_from_sub_path=true
-
-[auth]
-disable_login_form=true
-disable_signout_menu=true
-
-[auth.anonymous]
-enabled=true
-org_name=Main Org.
-org_role=Viewer
-
-[users]
-default_theme=light
-viewers_can_edit=true
-
-[security]
-allow_embedding=true
-EOF
-
-  mkdir -p $VOLUMES_ROOT/grafana/lib
-  chmod a+rwx $VOLUMES_ROOT/grafana/lib
-}
-
-initializeO11y() {
-  initializePrometheus
-  initializePromtail
-  initializeLoki
-  initializeGrafana
-}
 
 initializeConfigStorage
 initializeApiServer
 initializeController
-initializePilot
-initializeGateway
-initializeO11y