Merge pull request #11152 from owncloud/run-test-on-kubernetes

nirajacharya2 · web-flow · commit 22252371094d · 2025-07-24T12:15:16.000+05:45
[tests-only][full-ci] run tests on kubernetes
diff --git a/.drone.star b/.drone.star
@@ -138,17 +138,23 @@ config = {
             ],
             "skip": False,
         },
-        "davOperations": {
+        "davOperations1": {
             "suites": [
                 "apiSpacesDavOperation",
-                "apiArchiver",
                 "apiDownloads",
-                "apiActivities",
                 "apiAsyncUpload",
                 "apiDepthInfinity",
             ],
             "skip": False,
         },
+        "davOperations2": {
+            "suites": [
+                "apiArchiver",
+                "apiActivities",
+            ],
+            "skip": False,
+            "k3d": True,
+        },
         "groupAndSearch1": {
             "suites": [
                 "apiSearch1",
@@ -1068,6 +1074,7 @@ def localApiTestPipeline(ctx):
         "collaborationServiceNeeded": False,
         "extraCollaborationEnvironment": {},
         "withRemotePhp": with_remote_php,
+        "k3d": False,
     }
 
     if "localApiTests" in config:
@@ -1078,6 +1085,11 @@ def localApiTestPipeline(ctx):
                     params[item] = matrix[item] if item in matrix else defaults[item]
                 for storage in params["storages"]:
                     for run_with_remote_php in params["withRemotePhp"]:
+                        run_on_k3d = params["k3d"] and ctx.build.event == "cron"
+                        ocis_url = OCIS_URL
+                        if run_on_k3d:
+                            ocis_url = "https://%s" % OCIS_SERVER_NAME
+
                         pipeline = {
                             "kind": "pipeline",
                             "type": "docker",
@@ -1090,18 +1102,19 @@ def localApiTestPipeline(ctx):
                                      restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin") +
                                      (tikaService() if params["tikaNeeded"] else []) +
                                      (waitForServices("online-offices", ["collabora:9980", "onlyoffice:443", "fakeoffice:8080"]) if params["collaborationServiceNeeded"] else []) +
-                                     ocisServer(storage, extra_server_environment = params["extraServerEnvironment"], with_wrapper = True, tika_enabled = params["tikaNeeded"], volumes = ([stepVolumeOcisStorage] if name.startswith("cli") else [])) +
+                                     (waitK3sCluster() + deployOcis() + waitForOcis(ocis_url = ocis_url) + ocisServicePods() if run_on_k3d else ocisServer(storage, extra_server_environment = params["extraServerEnvironment"], with_wrapper = True, tika_enabled = params["tikaNeeded"], volumes = ([stepVolumeOcisStorage]))) +
                                      (waitForClamavService() if params["antivirusNeeded"] else []) +
                                      (waitForEmailService() if params["emailNeeded"] else []) +
                                      (ocisServer(storage, deploy_type = "federation", extra_server_environment = params["extraServerEnvironment"]) if params["federationServer"] else []) +
                                      ((wopiCollaborationService("fakeoffice") + wopiCollaborationService("collabora") + wopiCollaborationService("onlyoffice")) if params["collaborationServiceNeeded"] else []) +
                                      (ocisHealthCheck("wopi", ["wopi-collabora:9304", "wopi-onlyoffice:9304", "wopi-fakeoffice:9304"]) if params["collaborationServiceNeeded"] else []) +
-                                     localApiTests(ctx, name, params["suites"], storage, params["extraEnvironment"], run_with_remote_php) +
+                                     localApiTests(ctx, name, params["suites"], storage, params["extraEnvironment"], run_with_remote_php, ocis_url = ocis_url) +
                                      apiTestFailureLog() +
-                                     generateCoverageFromAPITest(ctx, name),
+                                     (generateCoverageFromAPITest(ctx, name) if not run_on_k3d else []),
                             "services": (emailService() if params["emailNeeded"] else []) +
                                         (clamavService() if params["antivirusNeeded"] else []) +
-                                        ((fakeOffice() + collaboraService() + onlyofficeService()) if params["collaborationServiceNeeded"] else []),
+                                        ((fakeOffice() + collaboraService() + onlyofficeService()) if params["collaborationServiceNeeded"] else []) +
+                                        (k3sCluster() if run_on_k3d else []),
                             "depends_on": getPipelineNames(buildOcisBinaryForTesting(ctx)),
                             "trigger": {
                                 "ref": [
@@ -1155,12 +1168,12 @@ def generateCoverageFromAPITest(ctx, name):
         },
     ]
 
-def localApiTests(ctx, name, suites, storage = "ocis", extra_environment = {}, with_remote_php = False):
+def localApiTests(ctx, name, suites, storage = "ocis", extra_environment = {}, with_remote_php = False, ocis_url = OCIS_URL):
     test_dir = "%s/tests/acceptance" % dirs["base"]
     expected_failures_file = "%s/expected-failures-localAPI-on-%s-storage.md" % (test_dir, storage.upper())
 
     environment = {
-        "TEST_SERVER_URL": OCIS_URL,
+        "TEST_SERVER_URL": ocis_url,
         "TEST_SERVER_FED_URL": OCIS_FED_URL,
         "OCIS_REVA_DATA_ROOT": "%s" % (dirs["ocisRevaDataRoot"] if storage == "owncloud" else ""),
         "STORAGE_DRIVER": storage,
@@ -2617,19 +2630,9 @@ def ocisServer(storage = "ocis", volumes = [], depends_on = [], deploy_type = ""
             "%s/bin/ociswrapper serve --bin %s --url %s --admin-username admin --admin-password admin" % (dirs["ocisWrapper"], ocis_bin, environment["OCIS_URL"]),
         ]
 
-    wait_for_ocis = waitForServices("ocis", [OCIS_DOMAIN])[0]
-    if not external_idp:
-        wait_for_ocis = {
-            "name": "wait-for-%s" % (container_name),
-            "image": OC_CI_ALPINE,
-            "commands": [
-                # wait for ocis-server to be ready (5 minutes)
-                "timeout 300 bash -c 'while [ $(curl -sk -uadmin:admin " +
-                "%s/graph/v1.0/users/admin " % environment["OCIS_URL"] +
-                "-w %{http_code} -o /dev/null) != 200 ]; do sleep 1; done'",
-            ],
-            "depends_on": depends_on,
-        }
+    wait_for_ocis = waitForOcis(container_name, environment["OCIS_URL"], depends_on)
+    if external_idp:
+        wait_for_ocis = waitForServices("ocis", [OCIS_DOMAIN])
 
     commands = [
         "mkdir -p $GOCOVERDIR",
@@ -2638,19 +2641,16 @@ def ocisServer(storage = "ocis", volumes = [], depends_on = [], deploy_type = ""
         "cp tests/config/drone/app-registry.yaml /root/.ocis/config/app-registry.yaml",
     ] + (build_and_run_commands)
 
-    return [
-        {
-            "name": container_name,
-            "image": OC_CI_GOLANG,
-            "detach": True,
-            "environment": environment,
-            "user": user,
-            "commands": commands,
-            "volumes": volumes,
-            "depends_on": depends_on,
-        },
-        wait_for_ocis,
-    ]
+    return [{
+        "name": container_name,
+        "image": OC_CI_GOLANG,
+        "detach": True,
+        "environment": environment,
+        "user": user,
+        "commands": commands,
+        "volumes": volumes,
+        "depends_on": depends_on,
+    }] + wait_for_ocis
 
 def startOcisService(service = None, name = None, environment = {}, volumes = []):
     """
@@ -3576,6 +3576,19 @@ def k6LoadTests(ctx):
         },
     }]
 
+def waitForOcis(name = "ocis", ocis_url = OCIS_URL, depends_on = []):
+    return [{
+        "name": "wait-for-%s" % name,
+        "image": OC_CI_ALPINE,
+        "commands": [
+            # wait for ocis-server to be ready (5 minutes)
+            "timeout 300 bash -c 'while [ $(curl -sk -uadmin:admin " +
+            "%s/graph/v1.0/users/admin " % ocis_url +
+            "-w %{http_code} -o /dev/null) != 200 ]; do sleep 1; done'",
+        ],
+        "depends_on": depends_on,
+    }]
+
 def waitForServices(name, services = []):
     services = ",".join(services)
     return [{
@@ -3754,3 +3767,82 @@ def trivyScan(ctx):
             ],
         },
     }
+
+def k3sCluster():
+    return [{
+        "name": OCIS_SERVER_NAME,
+        "image": "ghcr.io/k3d-io/k3d:5-dind",
+        "user": "root",
+        "privileged": True,
+        "commands": [
+            "git clone --single-branch --branch main --depth 1 https://github.com/owncloud/ocis-charts.git",
+            "nohup dockerd-entrypoint.sh &",
+            "until docker ps 2>&1 > /dev/null; do sleep 1s; done",
+            # create cluster
+            "k3d cluster create drone --api-port %s:33199 " % OCIS_SERVER_NAME +
+            "-p '80:80@loadbalancer' -p '443:443@loadbalancer' " +
+            "--k3s-arg '--tls-san=k3d@server:*' --k3s-arg '--disable=metrics-server@server:*'",
+            # wait for services to be ready
+            "until kubectl get deployment coredns -n kube-system -o go-template='{{.status.availableReplicas}}' | grep -v -e '<no value>'; do sleep 1s; done",
+            "until kubectl get deployment traefik -n kube-system -o go-template='{{.status.availableReplicas}}' | grep -v -e '<no value>'; do sleep 1s; done",
+            "k3d kubeconfig get drone > kubeconfig-$${DRONE_BUILD_NUMBER}.yaml",
+            "chmod 0600 kubeconfig-$${DRONE_BUILD_NUMBER}.yaml",
+            "printf '@@@@@@@@@@@@@@@@@@@@@@@\n@@@@ k3d is ready @@@@\n@@@@@@@@@@@@@@@@@@@@@@@\n'",
+            # add dns rewrite rule
+            "kubectl create configmap coredns-custom --namespace kube-system " +
+            "--from-literal='rewritehost.override=rewrite name exact %s host.k3d.internal'" % OCIS_SERVER_NAME,
+            "kubectl -n kube-system rollout restart deployment coredns",
+            # watch events
+            "kubectl get events -Aw",
+        ],
+    }]
+
+def waitK3sCluster():
+    return [{
+        "name": "wait-cluster",
+        "image": "docker.io/bitnami/kubectl:1.31",
+        "user": "root",
+        "commands": [
+            "export KUBECONFIG=kubeconfig-$${DRONE_BUILD_NUMBER}.yaml",
+            "until test -f $${KUBECONFIG}; do sleep 1s; done",
+            "kubectl config view",
+            "kubectl get pods -A",
+        ],
+    }]
+
+def deployOcis():
+    return [{
+        "name": "deploy-ocis",
+        "image": "owncloudci/golang:latest",
+        "commands": [
+            "mv %s/tests/config/drone/k3s/values.yaml %s/ocis-charts/charts/ocis/ci/deployment-values.yaml" % (dirs["base"], dirs["base"]),
+            "cp -r %s/tests/config/drone/k3s/authbasic %s/ocis-charts/charts/ocis/templates/" % (dirs["base"], dirs["base"]),
+            "cd %s/ocis-charts" % dirs["base"],
+            "sed -i '/{{- define \"ocis.basicServiceTemplates\" -}}/a\\\\  {{- $_ := set .scope \"appNameAuthBasic\" \"authbasic\" -}}' ./charts/ocis/templates/_common/_tplvalues.tpl",
+            "sed -i '/- name: IDM_ADMIN_PASSWORD/{n;N;N;N;d;}' ./charts/ocis/templates/idm/deployment.yaml",
+            "sed -i '/- name: IDM_ADMIN_PASSWORD/a\\\\\\n              value: \"admin\"' ./charts/ocis/templates/idm/deployment.yaml",
+            "sed -i '/- name: PROXY_HTTP_ADDR/i\\\\            - name: PROXY_ENABLE_BASIC_AUTH\\\n              value: \"true\"' ./charts/ocis/templates/proxy/deployment.yaml",
+            "export KUBECONFIG=%s/kubeconfig-$${DRONE_BUILD_NUMBER}.yaml" % dirs["base"],
+            "make helm-install-atomic",
+        ],
+        "volumes": [
+            {
+                "name": "gopath",
+                "path": "/go",
+            },
+        ],
+    }]
+
+def ocisServicePods():
+    return [{
+        "name": "ocis-pods",
+        "image": "ghcr.io/k3d-io/k3d:5-dind",
+        "user": "root",
+        "commands": [
+            "export KUBECONFIG=kubeconfig-$${DRONE_BUILD_NUMBER}.yaml",
+            "until test -f $${KUBECONFIG}; do sleep 1s; done",
+            "kubectl get pods -A",
+            "kubectl get ingress -A",
+            "kubectl get svc -A",
+        ],
+    }]
diff --git a/tests/config/drone/k3s/authbasic/deployment.yaml b/tests/config/drone/k3s/authbasic/deployment.yaml
@@ -0,0 +1,102 @@
+{{- include "ocis.basicServiceTemplates" (dict "scope" . "appName" "appNameAuthBasic" "appNameSuffix" "") -}}
+apiVersion: apps/v1
+kind: Deployment
+{{ include "ocis.metadata" . }}
+spec:
+  {{- include "ocis.selector" . | nindent 2 }}
+  {{- if and (not .Values.autoscaling.enabled) (.Values.replicas) }}
+  replicas: {{ .Values.replicas }}
+  {{- end }}
+  {{- include "ocis.deploymentStrategy" . | nindent 2 }}
+  template:
+    {{- include "ocis.templateMetadata" (dict "scope" $ "configCheck" false) | nindent 4 }}
+    spec:
+      {{- include "ocis.affinity" $ | nindent 6 }}
+      {{- include "ocis.securityContextAndtopologySpreadConstraints" . | nindent 6 }}
+      {{- include "ocis.priorityClassName" $.priorityClassName | nindent 6 }}
+      {{- include "ocis.hostAliases" $ | nindent 6 }}
+      nodeSelector: {{ toYaml $.nodeSelector | nindent 8 }}
+      containers:
+        - name: {{ .appName }}
+          {{- include "ocis.image" $ | nindent 10 }}
+          command: ["ocis"]
+          args: ["auth-basic", "server"]
+          {{- include "ocis.containerSecurityContext" . | nindent 10 }}
+          env:
+            {{- include "ocis.serviceRegistry" . | nindent 12 }}
+
+            - name: AUTH_BASIC_LOG_COLOR
+              value: {{ .Values.logging.color | quote }}
+            - name: AUTH_BASIC_LOG_LEVEL
+              value: {{ .Values.logging.level | quote }}
+            - name: AUTH_BASIC_LOG_PRETTY
+              value: {{ .Values.logging.pretty | quote }}
+
+            - name: AUTH_BASIC_TRACING_ENABLED
+              value: "{{ .Values.tracing.enabled }}"
+            - name: AUTH_BASIC_TRACING_TYPE
+              value: {{ .Values.tracing.type | quote }}
+            - name: AUTH_BASIC_TRACING_ENDPOINT
+              value: {{ .Values.tracing.endpoint | quote }}
+            - name: AUTH_BASIC_TRACING_COLLECTOR
+              value: {{ .Values.tracing.collector | quote }}
+
+            - name: AUTH_BASIC_DEBUG_PPROF
+              value: {{ .Values.debug.profiling | quote }}
+
+            # ----------------------------------------------------
+            - name: OCIS_URL
+              value: "https://{{ .Values.externalDomain }}"
+
+            - name: AUTH_BASIC_LDAP_URI
+              value: ldaps://{{ .appNameIdm }}:9235
+
+            - name: AUTH_BASIC_LDAP_CACERT
+              value: /etc/ocis/ldap-ca/ldap-ca.crt
+
+            - name: AUTH_BASIC_LDAP_BIND_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "secrets.ldapBindSecret" . }}
+                  key: reva-ldap-bind-password
+            # ----------------------------------------------------
+
+            - name: AUTH_BASIC_GRPC_ADDR
+              value: 0.0.0.0:9146
+            - name: AUTH_BASIC_DEBUG_ADDR
+              value: 0.0.0.0:9147
+
+            - name: AUTH_BASIC_JWT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "secrets.jwtSecret" . }}
+                  key: jwt-secret
+
+            {{- include "ocis.caEnv" $ | nindent 12}}
+
+          {{- include "ocis.livenessProbe" . | nindent 10 }}
+
+          resources: {{ toYaml .resources | nindent 12 }}
+
+          ports:
+            - name: grpc
+              containerPort: 9146
+            - name: metrics-debug
+              containerPort: 9147
+
+          volumeMounts:
+            - name: tmp-volume
+              mountPath: /tmp
+            - name: ldap-ca
+              mountPath: /etc/ocis/ldap-ca
+              readOnly: true
+            {{- include "ocis.caPath" $ | nindent 12}}
+
+      {{- include "ocis.imagePullSecrets" $ | nindent 6 }}
+      volumes:
+        - name: tmp-volume
+          emptyDir: {}
+        - name: ldap-ca
+          secret:
+            secretName: {{ include "secrets.ldapCASecret" . }}
+        {{- include "ocis.caVolume" $ | nindent 8}}
diff --git a/tests/config/drone/k3s/authbasic/hpa.yaml b/tests/config/drone/k3s/authbasic/hpa.yaml
@@ -0,0 +1,3 @@
+{{- include "ocis.basicServiceTemplates" (dict "scope" . "appName" "appNameAuthBasic" "appNameSuffix" "") -}}
+{{- $_ := set . "autoscaling" (default (default (dict) .Values.autoscaling) .Values.services.authservice.autoscaling) -}}
+{{ include "ocis.hpa" . }}
diff --git a/tests/config/drone/k3s/authbasic/pdb.yaml b/tests/config/drone/k3s/authbasic/pdb.yaml
@@ -0,0 +1,2 @@
+{{- include "ocis.basicServiceTemplates" (dict "scope" . "appName" "appNameAuthBasic" "appNameSuffix" "") -}}
+{{ include "ocis.pdb" . }}
diff --git a/tests/config/drone/k3s/authbasic/services.yaml b/tests/config/drone/k3s/authbasic/services.yaml
@@ -0,0 +1,22 @@
+{{- include "ocis.basicServiceTemplates" (dict "scope" . "appName" "appNameAuthBasic" "appNameSuffix" "") -}}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .appName }}
+  namespace: {{ template "ocis.namespace" . }}
+  labels:
+    app: {{ .appName }}
+    ocis-metrics: enabled
+    {{- include "ocis.labels" . | nindent 4 }}
+spec:
+  selector:
+    app: {{ .appName }}
+  ports:
+    - name: grpc
+      port: 9146
+      protocol: TCP
+      appProtocol: {{ .Values.service.appProtocol.grpc | quote}}
+    - name: metrics-debug
+      port: 9147
+      protocol: TCP
+      appProtocol: {{ .Values.service.appProtocol.http | quote}}
diff --git a/tests/config/drone/k3s/values.yaml b/tests/config/drone/k3s/values.yaml

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+{{- include "ocis.basicServiceTemplates" (dict "scope" . "appName" "appNameAuthBasic" "appNameSuffix" "") -}}`
	`2`	`+{{- $_ := set . "autoscaling" (default (default (dict) .Values.autoscaling) .Values.services.authservice.autoscaling) -}}`
	`3`	`+{{ include "ocis.hpa" . }}`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+{{- include "ocis.basicServiceTemplates" (dict "scope" . "appName" "appNameAuthBasic" "appNameSuffix" "") -}}`
	`2`	`+{{ include "ocis.pdb" . }}`