Support use a unified ingress resource (skypilot-org#8532)

* Support use a unified ingress resource

Signed-off-by: Aylei <rayingecho@gmail.com>

* Doc

Signed-off-by: Aylei <rayingecho@gmail.com>

* Adopt suggestion

Signed-off-by: Aylei <rayingecho@gmail.com>

---------

Signed-off-by: Aylei <rayingecho@gmail.com>

Author: aylei

charts/skypilot/templates/grafana-ingress.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.grafana.enabled .Values.grafana.ingress.enableAuthedIngress }}
+{{- if and (not .Values.ingress.unified) .Values.grafana.enabled .Values.grafana.ingress.enableAuthedIngress }}
 {{- $fullName := include "skypilot.fullname" . -}}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
@@ -23,6 +23,9 @@ metadata:
     {{- end }}
     nginx.ingress.kubernetes.io/configuration-snippet: |
       proxy_set_header X-WEBAUTH-USER admin;
+    {{- if .Values.grafana.ingress.annotations }}
+    {{- toYaml .Values.grafana.ingress.annotations | nindent 4 }}
+    {{- end }}
 spec:
   ingressClassName: {{ .Values.grafana.ingress.ingressClassName }}
   rules:

charts/skypilot/templates/ingress.yaml

@@ -4,6 +4,7 @@
 {{- $useNewIngressClass := or (gt ($kubeVersion | int) 1) (and (eq ($kubeVersion | int) 1) (ge $kubeMinorVersion 18)) -}}
 {{- $enableBasicAuthInAPIServer := include "skypilot.enableBasicAuthInAPIServer" . | trim | eq "true" -}}
 {{- $fullName := include "skypilot.fullname" . -}}
+{{- $unifiedIngress := .Values.ingress.unified | default false -}}
 
 apiVersion: networking.k8s.io/v1
 kind: Ingress
@@ -82,4 +83,22 @@ spec:
             name: {{ $fullName }}-api-service
             port:
               number: 80
+      {{- if and $unifiedIngress .Values.grafana.enabled }}
+      - pathType: Prefix
+        path: /grafana
+        backend:
+          service:
+            name: {{ $fullName }}-grafana
+            port:
+              number: 80
+      {{- end }}
+      {{- if and $unifiedIngress (index .Values.ingress "oauth2-proxy" "enabled") }}
+      - pathType: Prefix
+        path: /oauth2
+        backend:
+          service:
+            name: {{ $fullName }}-oauth2-proxy
+            port:
+              number: 4180
+      {{- end }}
 {{- end }}

charts/skypilot/templates/oauth2-proxy-ingress.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.ingress.enabled (index .Values.ingress "oauth2-proxy" "enabled") }}
+{{- if and (not .Values.ingress.unified) .Values.ingress.enabled (index .Values.ingress "oauth2-proxy" "enabled") }}
 {{- $kubeVersion := .Capabilities.KubeVersion.Major -}}
 {{- $kubeMinorVersion := .Capabilities.KubeVersion.Minor | trimSuffix "+" | int -}}
 {{- $useNewIngressClass := or (gt ($kubeVersion | int) 1) (and (eq ($kubeVersion | int) 1) (ge $kubeMinorVersion 18)) -}}

charts/skypilot/tests/metrics_test.yaml

@@ -28,3 +28,33 @@ tests:
       - matchRegex:
           path: data["prometheus.yml"]
           pattern: "custom-name-api-service\\.NAMESPACE\\.svc\\.cluster\\.local:9090"
+
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/helm-unittest/helm-unittest/main/schema/helm-testsuite.json
+suite: grafana_ingress_annotations_test
+templates:
+  - templates/grafana-ingress.yaml
+tests:
+  - it: should render custom annotations when grafana.ingress.annotations is set
+    set:
+      ingress:
+        unified: false
+      grafana:
+        enabled: true
+        ingress:
+          enableAuthedIngress: true
+          ingressClassName: nginx
+          annotations:
+            example.com/foo: "bar"
+            example.com/baz: "qux"
+    asserts:
+      - hasDocuments:
+          count: 1
+      - isKind:
+          of: Ingress
+      - equal:
+          path: metadata.annotations["example.com/foo"]
+          value: "bar"
+      - equal:
+          path: metadata.annotations["example.com/baz"]
+          value: "qux"

charts/skypilot/values.schema.json

@@ -471,6 +471,9 @@
                             "type": "boolean"
                         }
                     }
+                },
+                "unified": {
+                    "type": "boolean"
                 }
             }
         },

charts/skypilot/values.yaml

@@ -279,6 +279,9 @@ storage:
 
 
 ingress:
+  # Whether to use a unified ingress resource for the API server and other services like Grafana.
+  # @schema type: [boolean]
+  unified: false
   enabled: true
   # Name of the secret containing basic auth credentials for ingress. If not specified, a new secret will be created using authCredentials
   # Example:
@@ -674,6 +677,9 @@ grafana:
     # If you set hosts to null, the Grafana Helm chart will use its default value ([chart-example.local]).
     # To match all hosts, set hosts to an empty array ([]).
     hosts: []
+    # Custom annotations for the ingress
+    annotations: null
+
   grafana.ini:
     server:
       domain: localhost

docs/source/reference/api-server/helm-values-spec.rst

@@ -112,6 +112,7 @@ Below is the available helm value keys and the default value of each key:
 
   :ref:`ingress <helm-values-ingress>`:
     :ref:`enabled <helm-values-ingress-enabled>`: true
+    :ref:`unified <helm-values-ingress-unified>`: false
     :ref:`authSecret <helm-values-ingress-authSecret>`: null
     :ref:`authCredentials <helm-values-ingress-authCredentials>`: null
     :ref:`host <helm-values-ingress-host>`: null
@@ -1252,6 +1253,20 @@ Default: ``true``
   ingress:
     enabled: true
 
+.. _helm-values-ingress-unified:
+
+``ingress.unified``
+^^^^^^^^^^^^^^^^^^^
+
+Use a single ingress resource for the API server and other auxiliary services. Dedicated ingresses for these services will be skipped, e.g. grafana and oauth2-proxy.
+
+Default: ``false``
+
+.. code-block:: yaml
+
+  ingress:
+    unified: false
+
 .. _helm-values-ingress-authSecret:
 
 ``ingress.authSecret``