Adding an optional deployment to increase AI pickup speed

.github/workflows/lint-test.yaml

@@ -95,6 +95,10 @@ jobs:
             helm_args: --namespace nextcloud --skip-clean-up --helm-extra-set-args "--create-namespace --values charts/nextcloud/test-values/imaginary.yaml"
             test: true
 
+          # test the helm chart with ai-worker enabled
+          - name: AI-Worker Enabled
+            helm_args: --namespace nextcloud --helm-extra-set-args "--create-namespace --values charts/nextcloud/test-values/ai-worker.yaml"
+
     steps:
       - name: Checkout
         uses: actions/checkout@v4

charts/nextcloud/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: nextcloud
-version: 6.6.9
+version: 6.8.0
 # renovate: image=docker.io/library/nextcloud
 appVersion: 30.0.6
 description: A file sharing server that puts the control and security of your own data back into your hands.

charts/nextcloud/README.md

@@ -41,6 +41,7 @@ helm install my-release nextcloud/nextcloud
 * [Running `occ` commands](#running-occ-commands)
     * [Putting Nextcloud into maintanence mode](#putting-nextcloud-into-maintanence-mode)
     * [Downloading models for recognize](#downloading-models-for-recognize)
+* [Improve AI task pickup speed](#improve-ai-task-pickup-speed)
 * [Backups](#backups)
 * [Upgrades](#upgrades)
 * [Troubleshooting](#troubleshooting)
@@ -763,6 +764,26 @@ kubectl exec $NEXTCLOUD_POD -- su -s /bin/sh www-data -c "php occ maintenance:mo
 # $NEXTCLOUD_POD should be the name of *your* nextcloud pod :)
 kubectl exec $NEXTCLOUD_POD -- su -s /bin/sh www-data -c "php occ recognize:download-models"
 ```
+# Improve AI task pickup speed
+We provide a deployment that filters and prioritizes background jobs for AI tasks. This results in faster task processing and can be scaled according to your needs by increasing the number of replicas in the deployment.
+
+Check out the [official Nextcloud AI docs](https://docs.nextcloud.com/server/latest/admin_manual/ai/overview.html#improve-ai-task-pickup-speed) for more information.
+
+| Parameter                        | Description                                                                            | Default           |
+|----------------------------------|----------------------------------------------------------------------------------------|-------------------|
+| `aiWorker.enabled`               | Start the ai-worker deployment                                                         | `false`           |
+| `aiWorker.replicaCount`          | Number of ai-worker pod replicas to deploy                                             | `1`               |
+| `aiWorker.useHostName`           | Set to `true` to use the host defined in nextcloud.host and `false` to use the service | `false`           |
+| `aiWorker.resources`             | ai-worker resources                                                                    | `{}`              |
+| `aiWorker.securityContext`       | Optional security context for the ai-worker container                                  | `{}`              |
+| `aiWorker.podSecurityContext`    | Optional security context for the ai-worker container                                  | `{}`              |
+| `aiWorker.affinity`              | ai-worker pod affinity                                                                 | `{}`              |
+| `aiWorker.tolerations`           | ai-worker pod tolerations                                                              | `[]`              |
+| `aiWorker.deploymentAnnotations` | ai-worker deployment annotations                                                       | `{}`              |
+| `aiWorker.deploymentLabels`      | ai-worker deployment labels                                                            | `{}`              |
+| `aiWorker.podAnnotations`        | ai-worker pod annotations                                                              | `{}`              |
+| `aiWorker.podLabels`             | ai-worker pod labels                                                                   | `{}`              |
+
 
 # Backups
 Check out the [official Nextcloud backup docs](https://docs.nextcloud.com/server/latest/admin_manual/maintenance/backup.html). For your files, if you're using persistent volumes, and you'd like to back up to s3 backed storage (such as minio), consider using [k8up](https://github.com/k8up-io/k8up) or [velero](https://github.com/vmware-tanzu/velero).

charts/nextcloud/templates/_helpers.tpl

@@ -343,6 +343,39 @@ Swift as primary object store env vars
 {{- end -}}
 
 
+{{/*
+Volumes for a pod, which needs access to nextcloud data.
+*/}}
+{{- define "nextcloud.volumes" -}}
+- name: nextcloud-main
+  {{- if .Values.persistence.enabled }}
+  persistentVolumeClaim:
+    claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{- else }}{{ template "nextcloud.fullname" . }}-nextcloud{{- end }}
+  {{- else }}
+  emptyDir: {}
+  {{- end }}
+{{- if and .Values.persistence.nextcloudData.enabled .Values.persistence.enabled }}
+- name: nextcloud-data
+  persistentVolumeClaim:
+    claimName: {{ if .Values.persistence.nextcloudData.existingClaim }}{{ .Values.persistence.nextcloudData.existingClaim }}{{- else }}{{ template "nextcloud.fullname" . }}-nextcloud-data{{- end }}
+{{- end }}
+{{- if .Values.nextcloud.configs }}
+- name: nextcloud-config
+  configMap:
+    name: {{ template "nextcloud.fullname" . }}-config
+{{- end }}
+{{- if .Values.nextcloud.phpConfigs }}
+- name: nextcloud-phpconfig
+  configMap:
+    name: {{ template "nextcloud.fullname" . }}-phpconfig
+{{- end }}
+{{- with .Values.nextcloud.extraVolumes }}
+{{- toYaml . }}
+{{- end }}
+{{- end -}}
+
+
+
 {{/*
 Create volume mounts for the nextcloud container as well as the cron sidecar container.
 */}}
@@ -374,11 +407,13 @@ Create volume mounts for the nextcloud container as well as the cron sidecar con
 - name: nextcloud-main
   mountPath: /var/www/html/themes
   subPath: {{ ternary "themes" (printf "%s/themes" .Values.nextcloud.persistence.subPath) (empty .Values.nextcloud.persistence.subPath) }}
+
 {{- range $key, $value := .Values.nextcloud.configs }}
 - name: nextcloud-config
   mountPath: /var/www/html/config/{{ $key }}
   subPath: {{ $key }}
-{{- end }}
+{{- end }}{{/* end-range configs */}}
+
 {{- if .Values.nextcloud.configs }}
 {{- range $key, $value := .Values.nextcloud.defaultConfigs }}
 {{- if $value }}
@@ -387,16 +422,19 @@ Create volume mounts for the nextcloud container as well as the cron sidecar con
   subPath: {{ $key }}
 {{- end }}
 {{- end }}
-{{- end }}
-{{- if .Values.nextcloud.extraVolumeMounts }}
-{{ toYaml .Values.nextcloud.extraVolumeMounts }}
-{{- end }}
+{{- end }}{{/* end-if .configs (also defaultConfigs beside config, see above) */}}
+
+{{- with .Values.nextcloud.extraVolumeMounts }}
+{{ toYaml . }}
+{{- end }}{{/* end-with extraVolumes*/}}
+
 {{- $nginxEnabled := .Values.nginx.enabled -}}
 {{- range $key, $value := .Values.nextcloud.phpConfigs }}
 - name: nextcloud-phpconfig
   mountPath: {{ $nginxEnabled | ternary (printf "/usr/local/etc/php-fpm.d/%s" $key | quote) (printf "/usr/local/etc/php/conf.d/%s" $key | quote) }}
   subPath: {{ $key }}
-{{- end }}
+{{- end }}{{/* end-range phpconfig (with nginx) */}}
+
 {{- end -}}
 
 

charts/nextcloud/templates/ai-worker.yaml

@@ -0,0 +1,76 @@
+{{- if .Values.aiWorker.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "nextcloud.fullname" . }}-ai-worker
+  labels:
+    app.kubernetes.io/name: {{ include "nextcloud.name" . }}
+    helm.sh/chart: {{ include "nextcloud.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/component: ai-worker
+    {{- with .Values.aiWorker.deploymentLabels }}
+    {{- toYaml . | nindent 8 }}
+    {{- end }}
+  {{- with .Values.aiWorker.deploymentAnnotations }}
+  annotations:
+    {{- toYaml . | nindent 8 }}
+  {{- end }}
+spec:
+  replicas: {{ .Values.aiWorker.replicaCount }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "nextcloud.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      app.kubernetes.io/component: ai-worker
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ include "nextcloud.name" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/component: ai-worker
+        {{- with .Values.aiWorker.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      annotations:
+        {{- with .Values.aiWorker.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      affinity:
+        {{- toYaml .Values.aiWorker.affinity | nindent 8 }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.aiWorker.podSecurityContext | nindent 8 }}
+      initContainers:
+        - name: wait-for-service
+          image: {{ .Values.aiWorker.image.repository }}:{{ .Values.aiWorker.image.tag }}
+          command:
+            - sh
+            - -c
+            - >-
+              until curl -sf {{- if .Values.aiWorker.useHostName }}https://{{ .Values.nextcloud.host }}/status.php{{- else }}http://{{ template "nextcloud.fullname" . }}:{{ .Values.service.port }}/status.php{{- end }};
+              do echo waiting for service; sleep 5;
+              done
+      containers:
+        - name: nextcloud-aiworker
+          image: {{ include "nextcloud.image" . }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command:
+            - sh
+            - "-c"
+            - |-
+              set -e; while true; do php occ background-job:worker -v -t 60 "OC\TaskProcessing\SynchronousBackgroundJob"; done
+          securityContext:
+            {{- toYaml .Values.aiWorker.securityContext | nindent 12 }}
+          resources:
+            {{- toYaml .Values.aiWorker.resources | nindent 12 }}
+          volumeMounts:
+            {{- include "nextcloud.volumeMounts" . | trim | nindent 12 }}
+      volumes:
+        {{- include "nextcloud.volumes" . | trim | nindent 8 }}
+      restartPolicy: Always
+{{- end }}

charts/nextcloud/templates/deployment.yaml

@@ -340,28 +340,7 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
-        - name: nextcloud-main
-          {{- if .Values.persistence.enabled }}
-          persistentVolumeClaim:
-            claimName: {{ if .Values.persistence.existingClaim }}{{ .Values.persistence.existingClaim }}{{- else }}{{ template "nextcloud.fullname" . }}-nextcloud{{- end }}
-          {{- else }}
-          emptyDir: {}
-          {{- end }}
-        {{- if and .Values.persistence.nextcloudData.enabled .Values.persistence.enabled }}
-        - name: nextcloud-data
-          persistentVolumeClaim:
-            claimName: {{ if .Values.persistence.nextcloudData.existingClaim }}{{ .Values.persistence.nextcloudData.existingClaim }}{{- else }}{{ template "nextcloud.fullname" . }}-nextcloud-data{{- end }}
-        {{- end }}
-        {{- if .Values.nextcloud.configs }}
-        - name: nextcloud-config
-          configMap:
-            name: {{ template "nextcloud.fullname" . }}-config
-        {{- end }}
-        {{- if .Values.nextcloud.phpConfigs }}
-        - name: nextcloud-phpconfig
-          configMap:
-            name: {{ template "nextcloud.fullname" . }}-phpconfig
-        {{- end }}
+        {{- include "nextcloud.volumes" . | trim | nindent 8 }}
         {{- if .Values.nginx.enabled }}
         - name: nextcloud-nginx-config
           configMap:
@@ -373,9 +352,6 @@ spec:
             name: {{ template "nextcloud.fullname" . }}-hooks
             defaultMode: 0o755
         {{- end }}
-        {{- with .Values.nextcloud.extraVolumes }}
-        {{- toYaml . | nindent 8 }}
-        {{- end }}
       securityContext:
         {{- with .Values.securityContext }}
         {{- toYaml . | nindent 8 }}

charts/nextcloud/test-values/ai-worker.yaml

@@ -0,0 +1,21 @@
+podLabels:
+  workload: nextcloud
+internalDatabase:
+  enabled: true
+persistence:
+  enabled: true
+aiWorker:
+  enabled: true
+  replicaCount: 1
+  securityContext:
+    runAsUser: 33
+    runAsGroup: 33
+    runAsNonRoot: false
+  useHostName: false
+  affinity:
+    podAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        - labelSelector:
+            matchLabels:
+              workload: nextcloud
+          topologyKey: "kubernetes.io/hostname"

charts/nextcloud/values.yaml

@@ -686,6 +686,78 @@ dnsConfig: {}
 #   - name: ndots
 #     value: "1"
 
+
+## Deployment to execute nextcloud-assistant related background tasks
+## ref: https://docs.nextcloud.com/server/latest/admin_manual/ai/overview.html#improve-ai-task-pickup-speed
+##
+aiWorker:
+  # -- Start / Deploy Worker for AI
+  # @section -- AI Worker
+  enabled: false
+
+  image:
+    # --
+    # @section -- AI Worker
+    repository: alpine/curl
+    # --
+    # @section -- AI Worker
+    tag: latest
+    # --
+    # @section -- AI Worker
+    pullPolicy: IfNotPresent
+
+  # -- Number of replicas to be deployed
+  # @section -- AI Worker
+  replicaCount: 1
+
+  # -- Set securityContext parameters. For example, you may need to define runAsNonRoot directive
+  # @section -- AI Worker
+  securityContext: {}
+  #   runAsUser: 33
+  #   runAsGroup: 33
+  #   runAsNonRoot: true
+  #   readOnlyRootFilesystem: true
+
+  # -- Set podSecurityContext parameters. For example, you may need to define fsGroup directive
+  # @section -- AI Worker
+  podSecurityContext: {}
+  #   fsGroup: 33
+
+  # -- Set affinity parameters. For example, you may need to define a podAffinity when using a storageClass that doesn't support ReadWriteMany
+  # @section -- AI Worker
+  affinity: {}
+  #   podAffinity:
+  #     requiredDuringSchedulingIgnoredDuringExecution:
+  #       - labelSelector:
+  #           matchLabels:
+  #            app: nextcloud
+  #         topologyKey: "kubernetes.io/hostname"
+
+  # -- Set tolerations parameters. For example, you may need to define a toleration for a specific node taint
+  # @section -- AI Worker
+  tolerations: []
+
+  # -- Set to true to use the hostname to check nextcloud for readiness
+  # @section -- AI Worker
+  useHostName: true
+
+  # -- ai-worker resources
+  # @section -- AI Worker
+  resources: {}
+
+  # -- Annotations on Deployment
+  # @section -- AI Worker
+  deploymentAnnotations: {}
+  # -- Labels on Deployment
+  # @section -- AI Worker
+  deploymentLabels: {}
+  # -- Annotations on Pod
+  # @section -- AI Worker
+  podAnnotations: {}
+  # -- Labels on Pod
+  # @section -- AI Worker
+  podLabels: {}
+
 imaginary:
   # -- Start Imgaginary
   enabled: false