From 74b8621b75ae3e6f5e68534142ad5a8154a5b348 Mon Sep 17 00:00:00 2001 From: bennsimon Date: Tue, 4 Apr 2023 10:56:01 +0300 Subject: [PATCH 01/11] add helm chart --- charts/fhir-data-pipes/.helmignore | 23 +++ charts/fhir-data-pipes/Chart.yaml | 26 +++ charts/fhir-data-pipes/README.md | 92 ++++++++++ charts/fhir-data-pipes/templates/NOTES.txt | 22 +++ charts/fhir-data-pipes/templates/_helpers.tpl | 74 ++++++++ .../fhir-data-pipes/templates/configmap.yaml | 18 ++ .../fhir-data-pipes/templates/deployment.yaml | 91 ++++++++++ charts/fhir-data-pipes/templates/hpa.yaml | 28 +++ charts/fhir-data-pipes/templates/ingress.yaml | 61 +++++++ charts/fhir-data-pipes/templates/pdb.yaml | 22 +++ charts/fhir-data-pipes/templates/service.yaml | 15 ++ .../templates/serviceaccount.yaml | 12 ++ .../templates/tests/test-connection.yaml | 15 ++ charts/fhir-data-pipes/templates/vpa.yaml | 20 +++ charts/fhir-data-pipes/values.yaml | 163 ++++++++++++++++++ 15 files changed, 682 insertions(+) create mode 100644 charts/fhir-data-pipes/.helmignore create mode 100644 charts/fhir-data-pipes/Chart.yaml create mode 100644 charts/fhir-data-pipes/README.md create mode 100644 charts/fhir-data-pipes/templates/NOTES.txt create mode 100644 charts/fhir-data-pipes/templates/_helpers.tpl create mode 100644 charts/fhir-data-pipes/templates/configmap.yaml create mode 100644 charts/fhir-data-pipes/templates/deployment.yaml create mode 100644 charts/fhir-data-pipes/templates/hpa.yaml create mode 100644 charts/fhir-data-pipes/templates/ingress.yaml create mode 100644 charts/fhir-data-pipes/templates/pdb.yaml create mode 100644 charts/fhir-data-pipes/templates/service.yaml create mode 100644 charts/fhir-data-pipes/templates/serviceaccount.yaml create mode 100644 charts/fhir-data-pipes/templates/tests/test-connection.yaml create mode 100644 charts/fhir-data-pipes/templates/vpa.yaml create mode 100644 charts/fhir-data-pipes/values.yaml diff --git a/charts/fhir-data-pipes/.helmignore b/charts/fhir-data-pipes/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/fhir-data-pipes/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/fhir-data-pipes/Chart.yaml b/charts/fhir-data-pipes/Chart.yaml new file mode 100644 index 000000000..329e383a7 --- /dev/null +++ b/charts/fhir-data-pipes/Chart.yaml @@ -0,0 +1,26 @@ +apiVersion: v2 +name: fhir-data-pipes +description: An open-source pipeline to transform data from a FHIR server (like HAPI, GCP FHIR store, or even OpenMRS) using the FHIR format into a data warehouse based on Apache Parquet files, or another FHIR server. +maintainers: + - name: opensrp + email: techops@ona.io +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.1.0" diff --git a/charts/fhir-data-pipes/README.md b/charts/fhir-data-pipes/README.md new file mode 100644 index 000000000..7d57df777 --- /dev/null +++ b/charts/fhir-data-pipes/README.md @@ -0,0 +1,92 @@ + +# Fhir-data-pipes + +An open-source pipeline to transform data from a FHIR server (like HAPI, GCP FHIR store, or even OpenMRS) using the FHIR format into a data warehouse based on Apache Parquet files, or another FHIR server. + +## TL;DR + +```bash +$ helm repo add opensrp-fhir-data-pipes https://fhir-data-pipes.helm.smartregister.org +$ helm install fhir-data-pipes opensp-fhir-data-pipes/fhir-data-pipes +``` + +## Introduction + +This chart bootstraps [fhir-data-pipes](https://github.com/onaio/fhir-data-pipes) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Kubernetes 1.12+ +- Helm 3.1.0 + +## Configuration + +The following table lists the configurable parameters of the Fhir-data-pipes chart and their default values. + +| Parameter | Description | Default | +|-------------------------------------------------------|-------------|-------------------------------------------------------------------------------------------------------| +| `replicaCount` | | `1` | +| `image.repository` | | `"onaio/fhir-data-pipes"` | +| `image.pullPolicy` | | `"IfNotPresent"` | +| `image.tag` | | `"master"` | +| `imagePullSecrets` | | `[]` | +| `nameOverride` | | `""` | +| `fullnameOverride` | | `""` | +| `serviceAccount.create` | | `true` | +| `serviceAccount.annotations` | | `{}` | +| `serviceAccount.name` | | `""` | +| `podAnnotations` | | `{}` | +| `podSecurityContext` | | `{}` | +| `securityContext` | | `{}` | +| `service.type` | | `"ClusterIP"` | +| `service.port` | | `8080` | +| `ingress.enabled` | | `false` | +| `ingress.className` | | `""` | +| `ingress.annotations` | | `{}` | +| `ingress.hosts` | | `[{"host": "fhir-data-pipes.local", "paths": [{"path": "/", "pathType": "ImplementationSpecific"}]}]` | +| `ingress.tls` | | `[]` | +| `resources` | | `null` | +| `autoscaling.enabled` | | `false` | +| `autoscaling.minReplicas` | | `1` | +| `autoscaling.maxReplicas` | | `100` | +| `autoscaling.targetCPUUtilizationPercentage` | | `80` | +| `nodeSelector` | | `{}` | +| `tolerations` | | `[]` | +| `affinity` | | `{}` | +| `recreatePodsWhenConfigMapChange` | | `true` | +| `livenessProbe.httpGet.path` | | `"/"` | +| `livenessProbe.httpGet.port` | | `"http"` | +| `readinessProbe.httpGet.path` | | `"/"` | +| `readinessProbe.httpGet.port` | | `"http"` | +| `hapi.postgres.databaseService` | | `"postgresql"` | +| `hapi.postgres.databaseHostName` | | `""` | +| `hapi.postgres.databasePort` | | `"5432"` | +| `hapi.postgres.databaseUser` | | `""` | +| `hapi.postgres.databasePassword` | | `""` | +| `hapi.postgres.databaseName` | | `""` | +| `thriftserver.hive.databaseService` | | `"hive2"` | +| `thriftserver.hive.databaseHostName` | | `"spark-thriftserver"` | +| `thriftserver.hive.databasePort` | | `"10000"` | +| `thriftserver.hive.databaseUser` | | `"hive"` | +| `thriftserver.hive.databasePassword` | | `""` | +| `thriftserver.hive.databaseName` | | `"default"` | +| `applicationConfig.fhirdata.fhirServerUrl` | | `""` | +| `applicationConfig.fhirdata.dbConfig` | | `"/app/config/hapi-postgres-config.json"` | +| `applicationConfig.fhirdata.dwhRootPrefix` | | `"/dwh/controller_DWH"` | +| `applicationConfig.fhirdata.thriftserverHiveConfig` | | `"/app/config/thrifter-hive-config.json"` | +| `applicationConfig.fhirdata.incrementalSchedule` | | `"* * * * * *"` | +| `applicationConfig.fhirdata.resourceList` | | `"PatientEncounterObservation"` | +| `applicationConfig.fhirdata.maxWorkers` | | `"10"` | +| `applicationConfig.fhirdata.createHiveResourceTables` | | `"true"` | +| `applicationConfig.fhirdata.hiveJdbcDriver` | | `"org.apache.hive.jdbc.HiveDriver"` | +| `initContainers` | | `null` | +| `extraVolumes` | | `null` | +| `extraVolumeMounts` | | `null` | +| `extraConfigMaps` | | `null` | +| `env` | | `null` | +| `pdb.enabled` | | `false` | +| `pdb.minAvailable` | | `""` | +| `pdb.maxUnavailable` | | `1` | +| `vpa.enabled` | | `false` | +| `vpa.updatePolicy.updateMode` | | `"Off"` | +| `vpa.resourcePolicy` | | `{}` | diff --git a/charts/fhir-data-pipes/templates/NOTES.txt b/charts/fhir-data-pipes/templates/NOTES.txt new file mode 100644 index 000000000..e0ae0a4e6 --- /dev/null +++ b/charts/fhir-data-pipes/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "fhir-data-pipes.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "fhir-data-pipes.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "fhir-data-pipes.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "fhir-data-pipes.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/charts/fhir-data-pipes/templates/_helpers.tpl b/charts/fhir-data-pipes/templates/_helpers.tpl new file mode 100644 index 000000000..299afb0e0 --- /dev/null +++ b/charts/fhir-data-pipes/templates/_helpers.tpl @@ -0,0 +1,74 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "fhir-data-pipes.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "fhir-data-pipes.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "fhir-data-pipes.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "fhir-data-pipes.labels" -}} +helm.sh/chart: {{ include "fhir-data-pipes.chart" . }} +{{ include "fhir-data-pipes.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "fhir-data-pipes.selectorLabels" -}} +app.kubernetes.io/name: {{ include "fhir-data-pipes.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "fhir-data-pipes.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "fhir-data-pipes.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Populate the pod annotations +*/}} +{{- define "fhir-data-pipes.podAnnotations" -}} +{{- range $index, $element:=.Values.podAnnotations }} +{{ $index }}: {{ $element | quote }} +{{- end }} +{{- if .Values.recreatePodsWhenConfigMapChange }} +checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} +{{- end }} +{{- end }} diff --git a/charts/fhir-data-pipes/templates/configmap.yaml b/charts/fhir-data-pipes/templates/configmap.yaml new file mode 100644 index 000000000..1d2ce8b3b --- /dev/null +++ b/charts/fhir-data-pipes/templates/configmap.yaml @@ -0,0 +1,18 @@ +{{ $scope := . }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "fhir-data-pipes.fullname" . }} + labels: + {{- include "fhir-data-pipes.labels" . | nindent 4 }} +data: + hapi-postgres-config-json: | + {{ tpl (.Values.hapi.postgres | toJson ) $scope}} + thriftserver-hive-config-json: | + {{ tpl ( .Values.thriftserver.hive | toJson ) $scope }} + application-config-yaml: | + {{ tpl ( .Values.applicationConfig | toYaml) $scope | nindent 8}} + {{ range $value := .Values.extraConfigMaps -}} + {{ $value.name }}: | + {{ tpl $value.contents $scope | nindent 8 }} + {{ end }} diff --git a/charts/fhir-data-pipes/templates/deployment.yaml b/charts/fhir-data-pipes/templates/deployment.yaml new file mode 100644 index 000000000..1a42f57ff --- /dev/null +++ b/charts/fhir-data-pipes/templates/deployment.yaml @@ -0,0 +1,91 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "fhir-data-pipes.fullname" . }} + labels: + {{- include "fhir-data-pipes.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "fhir-data-pipes.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- include "fhir-data-pipes.podAnnotations" . | indent 8 }} + labels: + {{- include "fhir-data-pipes.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "fhir-data-pipes.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + volumes: + - name: application-config-yaml + configMap: + name: {{ include "fhir-data-pipes.fullname" . }} + - name: hapi-postgres-config-json + configMap: + name: {{ include "fhir-data-pipes.fullname" . }} + - name: thriftserver-hive-config-json + configMap: + name: {{ include "fhir-data-pipes.fullname" . }} + {{- if .Values.extraVolumes }} + {{- tpl (toYaml .Values.extraVolumes) . | nindent 12 }} + {{- end }} + {{- if .Values.initContainers }} + initContainers: + {{- toYaml .Values.initContainers | nindent 12 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: SPRING_CONFIG_LOCATION + value: /app/config/application.yaml + {{- if .Values.env }} + {{- tpl (toYaml .Values.env) . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8080 + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: application-config-yaml + mountPath: /app/config/application.yaml + subPath: application-config-yaml + - name: hapi-postgres-config-json + mountPath: /app/config/hapi-postgres-config.json + subPath: hapi-postgres-config-json + - name: thriftserver-hive-config-json + mountPath: /app/config/thriftserver-hive-config.json + subPath: thriftserver-hive-config-json + {{- if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/fhir-data-pipes/templates/hpa.yaml b/charts/fhir-data-pipes/templates/hpa.yaml new file mode 100644 index 000000000..d204a6456 --- /dev/null +++ b/charts/fhir-data-pipes/templates/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "fhir-data-pipes.fullname" . }} + labels: + {{- include "fhir-data-pipes.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "fhir-data-pipes.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/charts/fhir-data-pipes/templates/ingress.yaml b/charts/fhir-data-pipes/templates/ingress.yaml new file mode 100644 index 000000000..97c7f8248 --- /dev/null +++ b/charts/fhir-data-pipes/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "fhir-data-pipes.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "fhir-data-pipes.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/fhir-data-pipes/templates/pdb.yaml b/charts/fhir-data-pipes/templates/pdb.yaml new file mode 100644 index 000000000..ac7b3952e --- /dev/null +++ b/charts/fhir-data-pipes/templates/pdb.yaml @@ -0,0 +1,22 @@ +{{ if .Values.pdb.enabled }} +{{- if semverCompare "<1.21-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: "policy/v1beta1" +{{- else -}} +apiVersion: "policy/v1" +{{- end }} +kind: PodDisruptionBudget +metadata: + name: {{ include "fhir-data-pipes.fullname" . }} + labels: + {{- include "fhir-data-pipes.labels" . | nindent 4 }} +spec: + {{- if .Values.pdb.minAvailable }} + minAvailable: {{ .Values.pdb.minAvailable }} + {{- end }} + {{- if .Values.pdb.maxUnavailable }} + maxUnavailable: {{ .Values.pdb.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- include "fhir-data-pipes.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/fhir-data-pipes/templates/service.yaml b/charts/fhir-data-pipes/templates/service.yaml new file mode 100644 index 000000000..7066c002b --- /dev/null +++ b/charts/fhir-data-pipes/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "fhir-data-pipes.fullname" . }} + labels: + {{- include "fhir-data-pipes.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "fhir-data-pipes.selectorLabels" . | nindent 4 }} diff --git a/charts/fhir-data-pipes/templates/serviceaccount.yaml b/charts/fhir-data-pipes/templates/serviceaccount.yaml new file mode 100644 index 000000000..a16cf087f --- /dev/null +++ b/charts/fhir-data-pipes/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "fhir-data-pipes.serviceAccountName" . }} + labels: + {{- include "fhir-data-pipes.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/fhir-data-pipes/templates/tests/test-connection.yaml b/charts/fhir-data-pipes/templates/tests/test-connection.yaml new file mode 100644 index 000000000..b249c9469 --- /dev/null +++ b/charts/fhir-data-pipes/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "fhir-data-pipes.fullname" . }}-test-connection" + labels: + {{- include "fhir-data-pipes.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "fhir-data-pipes.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/charts/fhir-data-pipes/templates/vpa.yaml b/charts/fhir-data-pipes/templates/vpa.yaml new file mode 100644 index 000000000..782df1d71 --- /dev/null +++ b/charts/fhir-data-pipes/templates/vpa.yaml @@ -0,0 +1,20 @@ + +{{ if .Values.vpa.enabled }} +apiVersion: "autoscaling.k8s.io/v1" +kind: VerticalPodAutoscaler +metadata: + name: {{ include "fhir-data-pipes.fullname" . }} + labels: + {{- include "fhir-data-pipes.labels" . | nindent 4 }} +spec: + targetRef: + apiVersion: "apps/v1" + kind: Deployment + name: {{ include "fhir-data-pipes.fullname" . }} + updatePolicy: + {{- toYaml .Values.vpa.updatePolicy | nindent 4 }} + {{- if .Values.vpa.resourcePolicy }} + resourcePolicy: + {{- toYaml .Values.vpa.resourcePolicy | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/fhir-data-pipes/values.yaml b/charts/fhir-data-pipes/values.yaml new file mode 100644 index 000000000..664095411 --- /dev/null +++ b/charts/fhir-data-pipes/values.yaml @@ -0,0 +1,163 @@ +# Default values for fhir-data-pipes. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: onaio/fhir-data-pipes + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "master" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} +# fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true +# runAsNonRoot: true +# runAsUser: 1000 + +service: + type: ClusterIP + port: 8080 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: fhir-data-pipes.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: +# We usually recommend not to specify default resources and to leave this as a conscious +# choice for the user. This also increases chances charts run on environments with little +# resources such as Minikube. If you do want to specify resources uncomment the following +# lines adjust them as necessary and remove the curly braces after 'resources:'. +# limits: +# memory: 1024Mi +# requests: +# cpu: 100m +# memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +recreatePodsWhenConfigMapChange: true + +livenessProbe: + httpGet: + path: / + port: http + +readinessProbe: + httpGet: + path: / + port: http + +hapi: + postgres: + databaseService: "postgresql" + databaseHostName: "" + databasePort: "5432" + databaseUser: "" + databasePassword: "" + databaseName: "" + +thriftserver: + hive: + databaseService: "hive2" + databaseHostName: "spark-thriftserver" + databasePort: "10000" + databaseUser: "hive" + databasePassword: "" + databaseName: "default" + +applicationConfig: + fhirdata: + fhirServerUrl: "" + dbConfig: "/app/config/hapi-postgres-config.json" + dwhRootPrefix: "/dwh/controller_DWH" + thriftserverHiveConfig: "/app/config/thrifter-hive-config.json" + incrementalSchedule: "* * * * * *" + resourceList: "PatientEncounterObservation" + maxWorkers: "10" + createHiveResourceTables: "true" + hiveJdbcDriver: "org.apache.hive.jdbc.HiveDriver" + +initContainers: + +extraVolumes: +# - name: some-name +# configMap: +# name: '{{ include "fhir-data-pipes.fullname" . }}' + + +extraVolumeMounts: +# - mountPath: /app/resources/some-file.json +# name: some-name +# subPath: some-configmap.json + +extraConfigMaps: +# - name: some-configmap.json +# contents: | +# { +# "entries": [ +# { +# } +# ] +# } + +env: +# - name: APP_LOGGING_LEVEL +# value: DEBUG +# - name: _JAVA_OPTIONS +# value: -Dlogging.level.org.springframework=TRACE + +pdb: + enabled: false + minAvailable: "" + maxUnavailable: 1 + +vpa: + enabled: false + updatePolicy: + updateMode: "Off" + resourcePolicy: {} From 966d368de924c0d4dc1980f35b65f8c121634abe Mon Sep 17 00:00:00 2001 From: bennsimon Date: Thu, 6 Apr 2023 11:51:51 +0300 Subject: [PATCH 02/11] replace deployment with statefulset enable pvc by default create provisions for sidecar --- .../fhir-data-pipes/templates/configmap.yaml | 2 ++ .../templates/headless-svc.yaml | 11 +++++++ charts/fhir-data-pipes/templates/hpa.yaml | 2 +- charts/fhir-data-pipes/templates/ingress.yaml | 3 ++ charts/fhir-data-pipes/templates/pvc.yaml | 21 +++++++++++++ charts/fhir-data-pipes/templates/service.yaml | 3 ++ .../{deployment.yaml => statefulset.yaml} | 21 ++++++++++++- charts/fhir-data-pipes/templates/vpa.yaml | 2 +- charts/fhir-data-pipes/values.yaml | 31 ++++++++++++++++++- 9 files changed, 92 insertions(+), 4 deletions(-) create mode 100644 charts/fhir-data-pipes/templates/headless-svc.yaml create mode 100644 charts/fhir-data-pipes/templates/pvc.yaml rename charts/fhir-data-pipes/templates/{deployment.yaml => statefulset.yaml} (81%) diff --git a/charts/fhir-data-pipes/templates/configmap.yaml b/charts/fhir-data-pipes/templates/configmap.yaml index 1d2ce8b3b..77f56212b 100644 --- a/charts/fhir-data-pipes/templates/configmap.yaml +++ b/charts/fhir-data-pipes/templates/configmap.yaml @@ -12,6 +12,8 @@ data: {{ tpl ( .Values.thriftserver.hive | toJson ) $scope }} application-config-yaml: | {{ tpl ( .Values.applicationConfig | toYaml) $scope | nindent 8}} + flink-conf-yaml: | + {{ tpl ( .Values.flink | toYaml) $scope | nindent 8}} {{ range $value := .Values.extraConfigMaps -}} {{ $value.name }}: | {{ tpl $value.contents $scope | nindent 8 }} diff --git a/charts/fhir-data-pipes/templates/headless-svc.yaml b/charts/fhir-data-pipes/templates/headless-svc.yaml new file mode 100644 index 000000000..b5bd72a91 --- /dev/null +++ b/charts/fhir-data-pipes/templates/headless-svc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ printf "%s-headless" (include "fhir-data-pipes.fullname" .) }} + labels: + {{- include "fhir-data-pipes.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.headless.type }} + clusterIP: None + selector: + {{- include "fhir-data-pipes.selectorLabels" . | nindent 4 }} diff --git a/charts/fhir-data-pipes/templates/hpa.yaml b/charts/fhir-data-pipes/templates/hpa.yaml index d204a6456..c4477aa9e 100644 --- a/charts/fhir-data-pipes/templates/hpa.yaml +++ b/charts/fhir-data-pipes/templates/hpa.yaml @@ -8,7 +8,7 @@ metadata: spec: scaleTargetRef: apiVersion: apps/v1 - kind: Deployment + kind: StatefulSet name: {{ include "fhir-data-pipes.fullname" . }} minReplicas: {{ .Values.autoscaling.minReplicas }} maxReplicas: {{ .Values.autoscaling.maxReplicas }} diff --git a/charts/fhir-data-pipes/templates/ingress.yaml b/charts/fhir-data-pipes/templates/ingress.yaml index 97c7f8248..c1ef5785d 100644 --- a/charts/fhir-data-pipes/templates/ingress.yaml +++ b/charts/fhir-data-pipes/templates/ingress.yaml @@ -58,4 +58,7 @@ spec: {{- end }} {{- end }} {{- end }} + {{- if .Values.ingress.extraRules }} + {{- toYaml .Values.ingress.extraRules | nindent 4 }} + {{- end }} {{- end }} diff --git a/charts/fhir-data-pipes/templates/pvc.yaml b/charts/fhir-data-pipes/templates/pvc.yaml new file mode 100644 index 000000000..eb82e9e1c --- /dev/null +++ b/charts/fhir-data-pipes/templates/pvc.yaml @@ -0,0 +1,21 @@ +{{- if .Values.pvc.enabled -}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "fhir-data-pipes.fullname" . }} + labels: + {{- include "fhir-data-pipes.labels" . | nindent 4 }} +spec: + accessModes: + {{- toYaml .Values.pvc.accessModes | nindent 4 }} + volumeMode: {{ .Values.pvc.volumeMode }} + resources: + {{- toYaml .Values.pvc.resources | nindent 4 }} + {{- if .Values.pvc.storageClassName }} + storageClassName: {{ .Values.pvc.storageClassName }} + {{- end }} + {{- if .Values.pvc.selector }} + selector: + {{- toYaml .Values.pvc.selector | nindent 4 }} + {{- end }} +{{- end -}} \ No newline at end of file diff --git a/charts/fhir-data-pipes/templates/service.yaml b/charts/fhir-data-pipes/templates/service.yaml index 7066c002b..eb8323073 100644 --- a/charts/fhir-data-pipes/templates/service.yaml +++ b/charts/fhir-data-pipes/templates/service.yaml @@ -11,5 +11,8 @@ spec: targetPort: http protocol: TCP name: http + {{- if .Values.service.extraPorts }} + {{- toYaml .Values.service.extraPorts | nindent 4 }} + {{- end }} selector: {{- include "fhir-data-pipes.selectorLabels" . | nindent 4 }} diff --git a/charts/fhir-data-pipes/templates/deployment.yaml b/charts/fhir-data-pipes/templates/statefulset.yaml similarity index 81% rename from charts/fhir-data-pipes/templates/deployment.yaml rename to charts/fhir-data-pipes/templates/statefulset.yaml index 1a42f57ff..a606a3ea4 100644 --- a/charts/fhir-data-pipes/templates/deployment.yaml +++ b/charts/fhir-data-pipes/templates/statefulset.yaml @@ -1,5 +1,5 @@ apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: {{ include "fhir-data-pipes.fullname" . }} labels: @@ -8,6 +8,7 @@ spec: {{- if not .Values.autoscaling.enabled }} replicas: {{ .Values.replicaCount }} {{- end }} + serviceName: {{ printf "%s-headless" (include "fhir-data-pipes.fullname" .) }} selector: matchLabels: {{- include "fhir-data-pipes.selectorLabels" . | nindent 6 }} @@ -35,6 +36,14 @@ spec: - name: thriftserver-hive-config-json configMap: name: {{ include "fhir-data-pipes.fullname" . }} + - name: flink-conf-yaml + configMap: + name: {{ include "fhir-data-pipes.fullname" . }} + {{- if .Values.pvc.enabled }} + - name: dwh-dir + persistentVolumeClaim: + claimName: {{ include "fhir-data-pipes.fullname" . }} + {{- end }} {{- if .Values.extraVolumes }} {{- tpl (toYaml .Values.extraVolumes) . | nindent 12 }} {{- end }} @@ -74,9 +83,19 @@ spec: - name: thriftserver-hive-config-json mountPath: /app/config/thriftserver-hive-config.json subPath: thriftserver-hive-config-json + - name: flink-conf-yaml + mountPath: /app/config/flink-conf.yaml + subPath: flink-conf-yaml + {{- if .Values.pvc.enabled }} + - name: dwh-dir + mountPath: /dwh + {{- end }} {{- if .Values.extraVolumeMounts }} {{- toYaml .Values.extraVolumeMounts | nindent 12 }} {{- end }} + {{- if .Values.sidecars }} + {{- toYaml .Values.sidecars | nindent 8 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/fhir-data-pipes/templates/vpa.yaml b/charts/fhir-data-pipes/templates/vpa.yaml index 782df1d71..3a5eb5dde 100644 --- a/charts/fhir-data-pipes/templates/vpa.yaml +++ b/charts/fhir-data-pipes/templates/vpa.yaml @@ -9,7 +9,7 @@ metadata: spec: targetRef: apiVersion: "apps/v1" - kind: Deployment + kind: StatefulSet name: {{ include "fhir-data-pipes.fullname" . }} updatePolicy: {{- toYaml .Values.vpa.updatePolicy | nindent 4 }} diff --git a/charts/fhir-data-pipes/values.yaml b/charts/fhir-data-pipes/values.yaml index 664095411..d0cc15cfe 100644 --- a/charts/fhir-data-pipes/values.yaml +++ b/charts/fhir-data-pipes/values.yaml @@ -39,7 +39,17 @@ securityContext: {} service: type: ClusterIP port: 8080 - + extraPorts: +# - port: 4040 +# targetPort: hive-webui +# protocol: TCP +# name: hive-webui +# - port: 10000 +# targetPort: hive +# protocol: TCP +# name: hive + headless: + type: ClusterIP ingress: enabled: false className: "" @@ -52,6 +62,8 @@ ingress: - path: / pathType: ImplementationSpecific tls: [] + extraRules: + # - secretName: chart-example-tls # hosts: # - chart-example.local @@ -92,6 +104,9 @@ readinessProbe: path: / port: http +flink: + execution.attached: false + hapi: postgres: databaseService: "postgresql" @@ -124,6 +139,8 @@ applicationConfig: initContainers: +sidecars: + extraVolumes: # - name: some-name # configMap: @@ -161,3 +178,15 @@ vpa: updatePolicy: updateMode: "Off" resourcePolicy: {} + +pvc: + enabled: true + volumeMode: Filesystem + storageClassName: + resources: + requests: + # update accordingly + storage: 20Gi + accessModes: + - ReadWriteOnce + selector: {} From f4b6163d28e46a7c0f922f46e71fa30b9f1087c7 Mon Sep 17 00:00:00 2001 From: bennsimon Date: Tue, 11 Apr 2023 10:07:03 +0300 Subject: [PATCH 03/11] update chart readme --- charts/fhir-data-pipes/README.md | 71 ++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/charts/fhir-data-pipes/README.md b/charts/fhir-data-pipes/README.md index 7d57df777..f8d76621f 100644 --- a/charts/fhir-data-pipes/README.md +++ b/charts/fhir-data-pipes/README.md @@ -40,11 +40,14 @@ The following table lists the configurable parameters of the Fhir-data-pipes cha | `securityContext` | | `{}` | | `service.type` | | `"ClusterIP"` | | `service.port` | | `8080` | +| `service.extraPorts` | | `null` | +| `service.headless.type` | | `"ClusterIP"` | | `ingress.enabled` | | `false` | | `ingress.className` | | `""` | | `ingress.annotations` | | `{}` | | `ingress.hosts` | | `[{"host": "fhir-data-pipes.local", "paths": [{"path": "/", "pathType": "ImplementationSpecific"}]}]` | | `ingress.tls` | | `[]` | +| `ingress.extraRules` | | `null` | | `resources` | | `null` | | `autoscaling.enabled` | | `false` | | `autoscaling.minReplicas` | | `1` | @@ -58,6 +61,7 @@ The following table lists the configurable parameters of the Fhir-data-pipes cha | `livenessProbe.httpGet.port` | | `"http"` | | `readinessProbe.httpGet.path` | | `"/"` | | `readinessProbe.httpGet.port` | | `"http"` | +| `flink.execution.attached` | | `false` | | `hapi.postgres.databaseService` | | `"postgresql"` | | `hapi.postgres.databaseHostName` | | `""` | | `hapi.postgres.databasePort` | | `"5432"` | @@ -80,6 +84,7 @@ The following table lists the configurable parameters of the Fhir-data-pipes cha | `applicationConfig.fhirdata.createHiveResourceTables` | | `"true"` | | `applicationConfig.fhirdata.hiveJdbcDriver` | | `"org.apache.hive.jdbc.HiveDriver"` | | `initContainers` | | `null` | +| `sidecars` | | `null` | | `extraVolumes` | | `null` | | `extraVolumeMounts` | | `null` | | `extraConfigMaps` | | `null` | @@ -90,3 +95,69 @@ The following table lists the configurable parameters of the Fhir-data-pipes cha | `vpa.enabled` | | `false` | | `vpa.updatePolicy.updateMode` | | `"Off"` | | `vpa.resourcePolicy` | | `{}` | +| `pvc.enabled` | | `true` | +| `pvc.volumeMode` | | `"Filesystem"` | +| `pvc.storageClassName` | | `null` | +| `pvc.resources.requests.storage` | | `"20Gi"` | +| `pvc.accessModes` | | `["ReadWriteOnce"]` | +| `pvc.selector` | | `{}` | + + +## Spark SQL (Thrift Server) as Sidecar +The chart provides the necessary configuration to set up additional containers on the StatefulSet. One such container could be the spark thrift server. Below is how one can set it up. +````yaml +--- +sidecars: + - name: spark-thrift-server + image: docker.io/bitnami/spark:3.3.2-debian-11-r12 + imagePullPolicy: IfNotPresent + ports: + - name: hive + containerPort: 10000 + - name: hive-webui + containerPort: 4040 + args: + - "/bin/sh" + - "-c" + - "sbin/start-thriftserver.sh" + - "--master spark://spark-master-svc:7077" + env: + - name: "HIVE_SERVER2_THRIFT_PORT" + value: "10000" + resources: + limits: + memory: 1024Mi + requests: + cpu: 250m + memory: 256Mi + volumeMounts: + - name: dwh-dir + mountPath: /dwh + + +# Creates a volume for parquet files. +pvc: + enabled: true + volumeMode: Filesystem + storageClassName: + resources: + requests: + storage: 20Gi + accessModes: + - ReadWriteOnce + selector: {} + +# One can expose the thrift server port using tcp services e.g. for nginx(https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/exposing-tcp-udp-services.md) +service: + type: ClusterIP + port: 8080 + extraPorts: + - port: 4040 + targetPort: hive-webui + protocol: TCP + name: hive-webui + - port: 10000 + targetPort: hive + protocol: TCP + name: hive +```` From c822a28d879ccc85e8105de87dd938fbf60bc573 Mon Sep 17 00:00:00 2001 From: bennsimon Date: Wed, 19 Apr 2023 10:49:43 +0300 Subject: [PATCH 04/11] add github action workflows for docker image publish and helm chart lint and publish --- .github/workflows/chart-lint-checker.yml | 33 ++++++++ .github/workflows/docker-publish.yml | 99 ++++++++++++++++++++++++ .github/workflows/publish-chart.yml | 64 +++++++++++++++ ct.yaml | 21 +++++ 4 files changed, 217 insertions(+) create mode 100644 .github/workflows/chart-lint-checker.yml create mode 100644 .github/workflows/docker-publish.yml create mode 100644 .github/workflows/publish-chart.yml create mode 100644 ct.yaml diff --git a/.github/workflows/chart-lint-checker.yml b/.github/workflows/chart-lint-checker.yml new file mode 100644 index 000000000..4c520477c --- /dev/null +++ b/.github/workflows/chart-lint-checker.yml @@ -0,0 +1,33 @@ +name: Lint and Test Charts + +on: pull_request + +jobs: + lint-test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up Helm + uses: azure/setup-helm@v1 + with: + version: v3.5.0 + - uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.0.1 + - name: Run chart-testing (list-changed) + id: list-changed + run: | + changed=$(ct list-changed --config ct.yaml) + if [[ -n "$changed" ]]; then + echo "::set-output name=changed::true" + fi + - name: Run chart-testing (lint) + run: ct lint --config ct.yaml + - name: Create kind cluster + uses: helm/kind-action@v1.1.0 + if: steps.list-changed.outputs.changed == 'true' diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 000000000..16ca28595 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,99 @@ +name: Docker + +on: + push: + # Publish `master` as Docker `master` tag. + # See also https://github.com/crazy-max/ghaction-docker-meta#basic + branches: + - master + + # Publish `v1.2.3` tags as releases. + tags: + - v* + + pull_request: + # Run Tests when changes are made to the Docker file + paths: + - 'Dockerfile' + + workflow_dispatch: + +jobs: + # Run image build test + test: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Run Build tests + run: docker build . --file Dockerfile + + push: + runs-on: ubuntu-latest + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' + + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Cache Docker layers + uses: actions/cache@v2.1.6 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Docker meta + id: docker_meta + uses: docker/metadata-action@v4 + with: + images: onaio/fhir-data-pipes + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Push to Docker Image Repositories + uses: docker/build-push-action@v4 + id: docker_build + with: + push: true + context: . + platforms: linux/amd64,linux/arm64 + tags: | + ${{ steps.docker_meta.outputs.tags }} + ghcr.io/${{ steps.docker_meta.outputs.tags }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/publish-chart.yml b/.github/workflows/publish-chart.yml new file mode 100644 index 000000000..7a8ac55db --- /dev/null +++ b/.github/workflows/publish-chart.yml @@ -0,0 +1,64 @@ +# Kindly refer to https://github.com/helm/chart-releaser-action + +name: Publish Charts + +on: + push: + branches: + - master + +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Install Helm + uses: azure/setup-helm@v3 + + - name: Run chart-releaser + uses: helm/chart-releaser-action@v1.5.0 # step that writes the latest chart versions (below) depends on this step writing the latest version as the first index in the entries. list in the index.yaml file + env: + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + - name: Install Python + uses: actions/setup-python@v4 + + - name: Install pip requirements + uses: BSFishy/pip-action@v1 + with: + packages: | + shyaml==0.6.2 + + - name: Checkout gh-pages + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: gh-pages + + - name: Record the latest chart versions + run: | + releaseDir="latest" + if [[ ! -d ${releaseDir} ]]; then + mkdir -p ${releaseDir} + fi + if [[ -f index.yaml ]]; then + charts=($(cat index.yaml | shyaml keys entries)) + for curChart in "${charts[@]}"; do + curChartVersion=$(cat index.yaml | shyaml get-value entries.${curChart}.0.version) + echo ${curChartVersion} > ${releaseDir}/${curChart} + done + fi + + - uses: EndBug/add-and-commit@v7 + with: + message: 'Set the latest the chart versions' + branch: gh-pages diff --git a/ct.yaml b/ct.yaml new file mode 100644 index 000000000..38cf832f3 --- /dev/null +++ b/ct.yaml @@ -0,0 +1,21 @@ +# +# Copyright 2021-2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# See https://github.com/helm/chart-testing#configuration +remote: origin +target-branch: master +chart-dirs: + - charts From 4405644328b413ac42f128c0fd9aa028a3e0969b Mon Sep 17 00:00:00 2001 From: bennsimon Date: Thu, 20 Apr 2023 09:26:41 +0300 Subject: [PATCH 05/11] add support for hive-site-xml configurations --- charts/fhir-data-pipes/README.md | 4 +- .../fhir-data-pipes/templates/configmap.yaml | 4 ++ .../templates/statefulset.yaml | 5 ++ charts/fhir-data-pipes/values.yaml | 51 +++++++++++++++++-- 4 files changed, 60 insertions(+), 4 deletions(-) diff --git a/charts/fhir-data-pipes/README.md b/charts/fhir-data-pipes/README.md index f8d76621f..275462a37 100644 --- a/charts/fhir-data-pipes/README.md +++ b/charts/fhir-data-pipes/README.md @@ -133,7 +133,9 @@ sidecars: volumeMounts: - name: dwh-dir mountPath: /dwh - + - name: hive-site-xml + mountPath: /opt/bitnami/spark/conf/hive-site.xml + subPath: hive-site-xml # Creates a volume for parquet files. pvc: diff --git a/charts/fhir-data-pipes/templates/configmap.yaml b/charts/fhir-data-pipes/templates/configmap.yaml index 77f56212b..0c7a7a220 100644 --- a/charts/fhir-data-pipes/templates/configmap.yaml +++ b/charts/fhir-data-pipes/templates/configmap.yaml @@ -14,6 +14,10 @@ data: {{ tpl ( .Values.applicationConfig | toYaml) $scope | nindent 8}} flink-conf-yaml: | {{ tpl ( .Values.flink | toYaml) $scope | nindent 8}} + {{- if .Values.hiveSiteConfig }} + hive-site-xml: | + {{ tpl (.Values.hiveSiteConfig) $scope | nindent 4 | trim }} + {{- end }} {{ range $value := .Values.extraConfigMaps -}} {{ $value.name }}: | {{ tpl $value.contents $scope | nindent 8 }} diff --git a/charts/fhir-data-pipes/templates/statefulset.yaml b/charts/fhir-data-pipes/templates/statefulset.yaml index a606a3ea4..316051abc 100644 --- a/charts/fhir-data-pipes/templates/statefulset.yaml +++ b/charts/fhir-data-pipes/templates/statefulset.yaml @@ -39,6 +39,11 @@ spec: - name: flink-conf-yaml configMap: name: {{ include "fhir-data-pipes.fullname" . }} + {{- if .Values.hiveSiteConfig }} + - name: hive-site-xml + configMap: + name: {{ include "fhir-data-pipes.fullname" . }} + {{- end }} {{- if .Values.pvc.enabled }} - name: dwh-dir persistentVolumeClaim: diff --git a/charts/fhir-data-pipes/values.yaml b/charts/fhir-data-pipes/values.yaml index d0cc15cfe..aade744d8 100644 --- a/charts/fhir-data-pipes/values.yaml +++ b/charts/fhir-data-pipes/values.yaml @@ -137,16 +137,63 @@ applicationConfig: createHiveResourceTables: "true" hiveJdbcDriver: "org.apache.hive.jdbc.HiveDriver" +# Uncomment below block to override the hive-site.xml +#hiveSiteConfig: | +# +# +# +# +# +# hive.server2.authentication +# LDAP +# +# +# hive.server2.authentication.ldap.url +# ldap://openldap.default.svc.cluster.local:389 +# +# +# hive.server2.authentication.ldap.baseDN +# ou=users,dc=ldapadmin,dc=labs,dc=example,dc=org +# +# + initContainers: sidecars: +# - name: spark-thrift-server +# image: docker.io/bitnami/spark:3.3.2-debian-11-r12 +# imagePullPolicy: IfNotPresent +# ports: +# - name: hive +# containerPort: 10000 +# - name: hive-webui +# containerPort: 4040 +# args: +# - "/bin/sh" +# - "-c" +# - "sbin/start-thriftserver.sh" +# - "--master spark://spark-master-svc:7077" +# env: +# - name: "HIVE_SERVER2_THRIFT_PORT" +# value: "10000" +# resources: +# limits: +# memory: 1024Mi +# requests: +# cpu: 250m +# memory: 256Mi +# volumeMounts: +# - name: dwh-dir +# mountPath: /dwh +# - name: hive-site-xml +# mountPath: /opt/bitnami/spark/conf/hive-site.xml +# subPath: hive-site-xml extraVolumes: # - name: some-name # configMap: # name: '{{ include "fhir-data-pipes.fullname" . }}' - extraVolumeMounts: # - mountPath: /app/resources/some-file.json # name: some-name @@ -165,8 +212,6 @@ extraConfigMaps: env: # - name: APP_LOGGING_LEVEL # value: DEBUG -# - name: _JAVA_OPTIONS -# value: -Dlogging.level.org.springframework=TRACE pdb: enabled: false From 000407a6f034c181e0fd68e7e230df8a3287ecd4 Mon Sep 17 00:00:00 2001 From: bennsimon Date: Thu, 20 Apr 2023 18:41:35 +0300 Subject: [PATCH 06/11] fix lint update readme --- charts/fhir-data-pipes/README.md | 20 ++++++++++++++++ charts/fhir-data-pipes/values.yaml | 37 +++++++++++++++--------------- 2 files changed, 39 insertions(+), 18 deletions(-) diff --git a/charts/fhir-data-pipes/README.md b/charts/fhir-data-pipes/README.md index 275462a37..5a612733f 100644 --- a/charts/fhir-data-pipes/README.md +++ b/charts/fhir-data-pipes/README.md @@ -101,6 +101,7 @@ The following table lists the configurable parameters of the Fhir-data-pipes cha | `pvc.resources.requests.storage` | | `"20Gi"` | | `pvc.accessModes` | | `["ReadWriteOnce"]` | | `pvc.selector` | | `{}` | +| `hiveSiteConfig` | | `null` | ## Spark SQL (Thrift Server) as Sidecar @@ -162,4 +163,23 @@ service: targetPort: hive protocol: TCP name: hive + +hiveSiteConfig: | + + + + + + hive.server2.authentication + LDAP + + + hive.server2.authentication.ldap.url + ldap://openldap.default.svc.cluster.local:389 + + + hive.server2.authentication.ldap.baseDN + ou=users,dc=ldapadmin,dc=labs,dc=example,dc=org + + ```` diff --git a/charts/fhir-data-pipes/values.yaml b/charts/fhir-data-pipes/values.yaml index aade744d8..3febb357c 100644 --- a/charts/fhir-data-pipes/values.yaml +++ b/charts/fhir-data-pipes/values.yaml @@ -137,25 +137,26 @@ applicationConfig: createHiveResourceTables: "true" hiveJdbcDriver: "org.apache.hive.jdbc.HiveDriver" -# Uncomment below block to override the hive-site.xml -#hiveSiteConfig: | -# -# +# Override the hive-site.xml by specifying the xml as string, see example below (ensure indentation is correct) refer to README.md. +hiveSiteConfig: +# hiveSiteConfig: | +# +# # -# -# -# hive.server2.authentication -# LDAP -# -# -# hive.server2.authentication.ldap.url -# ldap://openldap.default.svc.cluster.local:389 -# -# -# hive.server2.authentication.ldap.baseDN -# ou=users,dc=ldapadmin,dc=labs,dc=example,dc=org -# -# +# +# +# hive.server2.authentication +# LDAP +# +# +# hive.server2.authentication.ldap.url +# ldap://openldap.default.svc.cluster.local:389 +# +# +# hive.server2.authentication.ldap.baseDN +# ou=users,dc=ldapadmin,dc=labs,dc=example,dc=org +# +# initContainers: From 08ea8590e13acf2b18966561e121860a658347e9 Mon Sep 17 00:00:00 2001 From: bennsimon Date: Mon, 24 Apr 2023 09:53:27 +0300 Subject: [PATCH 07/11] add google license headers --- charts/fhir-data-pipes/Chart.yaml | 16 ++++++++++++++++ .../fhir-data-pipes/templates/configmap.yaml | 16 ++++++++++++++++ .../templates/headless-svc.yaml | 16 ++++++++++++++++ charts/fhir-data-pipes/templates/hpa.yaml | 18 +++++++++++++++++- charts/fhir-data-pipes/templates/ingress.yaml | 18 +++++++++++++++++- charts/fhir-data-pipes/templates/pdb.yaml | 16 ++++++++++++++++ charts/fhir-data-pipes/templates/pvc.yaml | 18 +++++++++++++++++- charts/fhir-data-pipes/templates/service.yaml | 16 ++++++++++++++++ .../templates/serviceaccount.yaml | 18 +++++++++++++++++- .../fhir-data-pipes/templates/statefulset.yaml | 16 ++++++++++++++++ .../templates/tests/test-connection.yaml | 16 ++++++++++++++++ charts/fhir-data-pipes/templates/vpa.yaml | 16 ++++++++++++++++ charts/fhir-data-pipes/values.yaml | 16 ++++++++++++++++ 13 files changed, 212 insertions(+), 4 deletions(-) diff --git a/charts/fhir-data-pipes/Chart.yaml b/charts/fhir-data-pipes/Chart.yaml index 329e383a7..e71bc5461 100644 --- a/charts/fhir-data-pipes/Chart.yaml +++ b/charts/fhir-data-pipes/Chart.yaml @@ -1,3 +1,19 @@ +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + apiVersion: v2 name: fhir-data-pipes description: An open-source pipeline to transform data from a FHIR server (like HAPI, GCP FHIR store, or even OpenMRS) using the FHIR format into a data warehouse based on Apache Parquet files, or another FHIR server. diff --git a/charts/fhir-data-pipes/templates/configmap.yaml b/charts/fhir-data-pipes/templates/configmap.yaml index 0c7a7a220..19327d87c 100644 --- a/charts/fhir-data-pipes/templates/configmap.yaml +++ b/charts/fhir-data-pipes/templates/configmap.yaml @@ -1,3 +1,19 @@ +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + {{ $scope := . }} apiVersion: v1 kind: ConfigMap diff --git a/charts/fhir-data-pipes/templates/headless-svc.yaml b/charts/fhir-data-pipes/templates/headless-svc.yaml index b5bd72a91..bbcf867aa 100644 --- a/charts/fhir-data-pipes/templates/headless-svc.yaml +++ b/charts/fhir-data-pipes/templates/headless-svc.yaml @@ -1,3 +1,19 @@ +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + apiVersion: v1 kind: Service metadata: diff --git a/charts/fhir-data-pipes/templates/hpa.yaml b/charts/fhir-data-pipes/templates/hpa.yaml index c4477aa9e..0a282d3a7 100644 --- a/charts/fhir-data-pipes/templates/hpa.yaml +++ b/charts/fhir-data-pipes/templates/hpa.yaml @@ -1,4 +1,20 @@ -{{- if .Values.autoscaling.enabled }} +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{ if .Values.autoscaling.enabled }} apiVersion: autoscaling/v2beta1 kind: HorizontalPodAutoscaler metadata: diff --git a/charts/fhir-data-pipes/templates/ingress.yaml b/charts/fhir-data-pipes/templates/ingress.yaml index c1ef5785d..c6eca914e 100644 --- a/charts/fhir-data-pipes/templates/ingress.yaml +++ b/charts/fhir-data-pipes/templates/ingress.yaml @@ -1,4 +1,20 @@ -{{- if .Values.ingress.enabled -}} +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{ if .Values.ingress.enabled }} {{- $fullName := include "fhir-data-pipes.fullname" . -}} {{- $svcPort := .Values.service.port -}} {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} diff --git a/charts/fhir-data-pipes/templates/pdb.yaml b/charts/fhir-data-pipes/templates/pdb.yaml index ac7b3952e..592d4bf22 100644 --- a/charts/fhir-data-pipes/templates/pdb.yaml +++ b/charts/fhir-data-pipes/templates/pdb.yaml @@ -1,3 +1,19 @@ +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + {{ if .Values.pdb.enabled }} {{- if semverCompare "<1.21-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: "policy/v1beta1" diff --git a/charts/fhir-data-pipes/templates/pvc.yaml b/charts/fhir-data-pipes/templates/pvc.yaml index eb82e9e1c..a31857c50 100644 --- a/charts/fhir-data-pipes/templates/pvc.yaml +++ b/charts/fhir-data-pipes/templates/pvc.yaml @@ -1,4 +1,20 @@ -{{- if .Values.pvc.enabled -}} +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{ if .Values.pvc.enabled }} apiVersion: v1 kind: PersistentVolumeClaim metadata: diff --git a/charts/fhir-data-pipes/templates/service.yaml b/charts/fhir-data-pipes/templates/service.yaml index eb8323073..51a62f9c2 100644 --- a/charts/fhir-data-pipes/templates/service.yaml +++ b/charts/fhir-data-pipes/templates/service.yaml @@ -1,3 +1,19 @@ +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + apiVersion: v1 kind: Service metadata: diff --git a/charts/fhir-data-pipes/templates/serviceaccount.yaml b/charts/fhir-data-pipes/templates/serviceaccount.yaml index a16cf087f..ce8ed6a79 100644 --- a/charts/fhir-data-pipes/templates/serviceaccount.yaml +++ b/charts/fhir-data-pipes/templates/serviceaccount.yaml @@ -1,4 +1,20 @@ -{{- if .Values.serviceAccount.create -}} +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{ if .Values.serviceAccount.create }} apiVersion: v1 kind: ServiceAccount metadata: diff --git a/charts/fhir-data-pipes/templates/statefulset.yaml b/charts/fhir-data-pipes/templates/statefulset.yaml index 316051abc..146bf4562 100644 --- a/charts/fhir-data-pipes/templates/statefulset.yaml +++ b/charts/fhir-data-pipes/templates/statefulset.yaml @@ -1,3 +1,19 @@ +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + apiVersion: apps/v1 kind: StatefulSet metadata: diff --git a/charts/fhir-data-pipes/templates/tests/test-connection.yaml b/charts/fhir-data-pipes/templates/tests/test-connection.yaml index b249c9469..187099169 100644 --- a/charts/fhir-data-pipes/templates/tests/test-connection.yaml +++ b/charts/fhir-data-pipes/templates/tests/test-connection.yaml @@ -1,3 +1,19 @@ +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + apiVersion: v1 kind: Pod metadata: diff --git a/charts/fhir-data-pipes/templates/vpa.yaml b/charts/fhir-data-pipes/templates/vpa.yaml index 3a5eb5dde..446e39002 100644 --- a/charts/fhir-data-pipes/templates/vpa.yaml +++ b/charts/fhir-data-pipes/templates/vpa.yaml @@ -1,3 +1,19 @@ +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + {{ if .Values.vpa.enabled }} apiVersion: "autoscaling.k8s.io/v1" diff --git a/charts/fhir-data-pipes/values.yaml b/charts/fhir-data-pipes/values.yaml index 3febb357c..c71432756 100644 --- a/charts/fhir-data-pipes/values.yaml +++ b/charts/fhir-data-pipes/values.yaml @@ -1,3 +1,19 @@ +# +# Copyright 2020-2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + # Default values for fhir-data-pipes. # This is a YAML-formatted file. # Declare variables to be passed into your templates. From 6ad61b01d2c73f6ce78f546f8aa3cf8b3ed7d2f1 Mon Sep 17 00:00:00 2001 From: bennsimon Date: Mon, 24 Apr 2023 10:10:46 +0300 Subject: [PATCH 08/11] cleanup --- charts/fhir-data-pipes/templates/pvc.yaml | 2 +- ct.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/fhir-data-pipes/templates/pvc.yaml b/charts/fhir-data-pipes/templates/pvc.yaml index a31857c50..97f220bc6 100644 --- a/charts/fhir-data-pipes/templates/pvc.yaml +++ b/charts/fhir-data-pipes/templates/pvc.yaml @@ -34,4 +34,4 @@ spec: selector: {{- toYaml .Values.pvc.selector | nindent 4 }} {{- end }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/ct.yaml b/ct.yaml index 38cf832f3..4e299ffd4 100644 --- a/ct.yaml +++ b/ct.yaml @@ -1,5 +1,5 @@ # -# Copyright 2021-2022 Google LLC +# Copyright 2021-2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From df861611329075c9e2f69a062d617acf4e3cf618 Mon Sep 17 00:00:00 2001 From: bennsimon Date: Mon, 24 Apr 2023 10:40:49 +0300 Subject: [PATCH 09/11] update docker meta tags --- .github/workflows/docker-publish.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 16ca28595..ff844e279 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -60,6 +60,12 @@ jobs: uses: docker/metadata-action@v4 with: images: onaio/fhir-data-pipes + tags: | + type=schedule + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + type=raw,value=${{ github.event.inputs.customTag }} - name: Login to DockerHub uses: docker/login-action@v2 From 4baf345d13513222564f93049d800a26a1dcc74f Mon Sep 17 00:00:00 2001 From: bennsimon Date: Mon, 24 Apr 2023 10:45:43 +0300 Subject: [PATCH 10/11] remove docker publish workflow --- .github/workflows/docker-publish.yml | 105 --------------------------- 1 file changed, 105 deletions(-) delete mode 100644 .github/workflows/docker-publish.yml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml deleted file mode 100644 index ff844e279..000000000 --- a/.github/workflows/docker-publish.yml +++ /dev/null @@ -1,105 +0,0 @@ -name: Docker - -on: - push: - # Publish `master` as Docker `master` tag. - # See also https://github.com/crazy-max/ghaction-docker-meta#basic - branches: - - master - - # Publish `v1.2.3` tags as releases. - tags: - - v* - - pull_request: - # Run Tests when changes are made to the Docker file - paths: - - 'Dockerfile' - - workflow_dispatch: - -jobs: - # Run image build test - test: - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' - - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - - - name: Run Build tests - run: docker build . --file Dockerfile - - push: - runs-on: ubuntu-latest - if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' - - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Cache Docker layers - uses: actions/cache@v2.1.6 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Docker meta - id: docker_meta - uses: docker/metadata-action@v4 - with: - images: onaio/fhir-data-pipes - tags: | - type=schedule - type=ref,event=branch - type=ref,event=tag - type=ref,event=pr - type=raw,value=${{ github.event.inputs.customTag }} - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Push to Docker Image Repositories - uses: docker/build-push-action@v4 - id: docker_build - with: - push: true - context: . - platforms: linux/amd64,linux/arm64 - tags: | - ${{ steps.docker_meta.outputs.tags }} - ghcr.io/${{ steps.docker_meta.outputs.tags }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - - name: Move cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} From 10bf49187c91bdd743a544e3c1d569cc6be28ef6 Mon Sep 17 00:00:00 2001 From: bennsimon Date: Mon, 24 Apr 2023 11:10:00 +0300 Subject: [PATCH 11/11] update readme change onaio ref to google --- charts/fhir-data-pipes/README.md | 21 ++++++++++++++++----- charts/fhir-data-pipes/values.yaml | 6 +++--- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/charts/fhir-data-pipes/README.md b/charts/fhir-data-pipes/README.md index 5a612733f..62ba36c1e 100644 --- a/charts/fhir-data-pipes/README.md +++ b/charts/fhir-data-pipes/README.md @@ -6,13 +6,13 @@ An open-source pipeline to transform data from a FHIR server (like HAPI, GCP FHI ## TL;DR ```bash -$ helm repo add opensrp-fhir-data-pipes https://fhir-data-pipes.helm.smartregister.org -$ helm install fhir-data-pipes opensp-fhir-data-pipes/fhir-data-pipes +$ helm repo add google-fhir-data-pipes https://google.github.io/fhir-data-pipes/ +$ helm install fhir-data-pipes google-fhir-data-pipes/fhir-data-pipes ``` ## Introduction -This chart bootstraps [fhir-data-pipes](https://github.com/onaio/fhir-data-pipes) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. +This chart bootstraps [fhir-data-pipes](https://github.com/google/fhir-data-pipes) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. ## Prerequisites @@ -26,7 +26,7 @@ The following table lists the configurable parameters of the Fhir-data-pipes cha | Parameter | Description | Default | |-------------------------------------------------------|-------------|-------------------------------------------------------------------------------------------------------| | `replicaCount` | | `1` | -| `image.repository` | | `"onaio/fhir-data-pipes"` | +| `image.repository` | | `"google/fhir-data-pipes"` | | `image.pullPolicy` | | `"IfNotPresent"` | | `image.tag` | | `"master"` | | `imagePullSecrets` | | `[]` | @@ -105,7 +105,7 @@ The following table lists the configurable parameters of the Fhir-data-pipes cha ## Spark SQL (Thrift Server) as Sidecar -The chart provides the necessary configuration to set up additional containers on the StatefulSet. One such container could be the spark thrift server. Below is how one can set it up. +The chart provides the necessary configuration to set up additional containers on the StatefulSet. One such container is the spark thrift server. Below is how one can set it up. ````yaml --- sidecars: @@ -164,6 +164,17 @@ service: protocol: TCP name: hive +# For LDAP ensure that user has the attribute `uid` on its DN. +# As of now the hive jdbc credentials are not parsed on the pipeline code, as a workaround append the credentials on the `databaseName` credentials as follows: +#thriftserver: +# hive: +# databaseService: "hive2" +# databaseHostName: "fhir-data-pipes" +# databasePort: "10000" +# databaseUser: "user" (not used as of now) +# databasePassword: "password" (not used as of now) +# databaseName: "default;user=;password=" + hiveSiteConfig: | diff --git a/charts/fhir-data-pipes/values.yaml b/charts/fhir-data-pipes/values.yaml index c71432756..1484f005f 100644 --- a/charts/fhir-data-pipes/values.yaml +++ b/charts/fhir-data-pipes/values.yaml @@ -21,7 +21,7 @@ replicaCount: 1 image: - repository: onaio/fhir-data-pipes + repository: google/fhir-data-pipes pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. tag: "master" @@ -146,9 +146,9 @@ applicationConfig: fhirServerUrl: "" dbConfig: "/app/config/hapi-postgres-config.json" dwhRootPrefix: "/dwh/controller_DWH" - thriftserverHiveConfig: "/app/config/thrifter-hive-config.json" + thriftserverHiveConfig: "/app/config/thriftserver-hive-config.json" incrementalSchedule: "* * * * * *" - resourceList: "PatientEncounterObservation" + resourceList: "Patient,Encounter,Observation" maxWorkers: "10" createHiveResourceTables: "true" hiveJdbcDriver: "org.apache.hive.jdbc.HiveDriver"