From 5762204bf7b6e3cbae0164e3c1b61ffaf78eef92 Mon Sep 17 00:00:00 2001 From: Laure Vancau Date: Thu, 23 Oct 2025 13:07:09 +0200 Subject: [PATCH 01/17] feat: build images and manifests --- .github/workflows/build-images-manifests.yml | 125 +++++++++++++++++++ tools/deploy/presidio/values.yaml | 2 +- 2 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build-images-manifests.yml diff --git a/.github/workflows/build-images-manifests.yml b/.github/workflows/build-images-manifests.yml new file mode 100644 index 0000000..319cdd9 --- /dev/null +++ b/.github/workflows/build-images-manifests.yml @@ -0,0 +1,125 @@ +name: Presidio Docker Build + +on: + pull_request: + branches: [ main ] + push: + branches: [ main ] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: read-all + +env: + REGISTRY_NAME: ghcr.io # SDSC ADD-ON + TAG: gha${{ github.run_number }} + +jobs: + build-platform-images: + name: Build ${{ matrix.image }} (${{ matrix.platform }}) + runs-on: ${{ matrix.runner }} + if: github.ref == 'refs/heads/main' + permissions: + id-token: write + contents: read + strategy: + matrix: + include: + - image: presidio-anonymizer + platform: linux/amd64 + runner: ubuntu-latest + - image: presidio-analyzer + platform: linux/amd64 + runner: ubuntu-latest + - image: presidio-image-redactor + platform: linux/amd64 + runner: ubuntu-latest + - image: presidio-anonymizer + platform: linux/arm64 + runner: ubuntu-24.04-arm + - image: presidio-analyzer + platform: linux/arm64 + runner: ubuntu-24.04-arm + - image: presidio-image-redactor + platform: linux/arm64 + runner: ubuntu-24.04-arm + steps: + # SDSC ADD-ON + - name: Get latest Presidio release tag + id: presidio_release + run: | + tag=$(curl -s https://api.github.com/repos/microsoft/presidio/releases/latest | jq -r .tag_name) + echo "tag=$tag" >> $GITHUB_OUTPUT + + # SDSC ADD-ON + - name: Checkout Presidio (latest release) + uses: actions/checkout@v5 + with: + repository: microsoft/presidio + ref: ${{ steps.presidio_release.outputs.tag }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + # SDSC ADD-ON + # https://github.com/docker/login-action + - name: Log in to the Container registry + uses: docker/login-action@v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and Push ${{ matrix.image }} for ${{ matrix.platform }} + run: | + # Create platform-specific tag + PLATFORM_TAG=$(echo "${{ matrix.platform }}" | sed 's/\//-/g') + docker buildx build \ + --platform ${{ matrix.platform }} \ + --push \ + --tag ${{ env.REGISTRY_NAME }}/${{ matrix.image }}:${{ env.TAG }}-${PLATFORM_TAG} \ + --cache-from type=registry,ref=${{ env.REGISTRY_NAME }}/public/${{ matrix.image }}:latest \ + --cache-to type=inline \ + ./${{ matrix.image }} + env: + REGISTRY_NAME: ${{ env.REGISTRY_NAME }} + TAG: ${{ env.TAG }} + + create-manifests: + name: Create Multi-Platform Manifests + runs-on: ubuntu-latest + needs: build-platform-images + if: github.ref == 'refs/heads/main' + permissions: + id-token: write + contents: read + steps: + # SDSC ADD-ON + # https://github.com/docker/login-action + - name: Log in to the Container registry + uses: docker/login-action@v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Create all multi-platform manifests + run: | + IMAGES=("presidio-anonymizer" "presidio-analyzer" "presidio-image-redactor") + + for image in "${IMAGES[@]}"; do + echo "Creating manifest for $image" + docker buildx imagetools create \ + --tag ${{ env.REGISTRY_NAME }}/${image}:${{ env.TAG }} \ + ${{ env.REGISTRY_NAME }}/${image}:${{ env.TAG }}-linux-amd64 \ + ${{ env.REGISTRY_NAME }}/${image}:${{ env.TAG }}-linux-arm64 + done + env: + REGISTRY_NAME: ${{ env.REGISTRY_NAME }} + TAG: ${{ env.TAG }} \ No newline at end of file diff --git a/tools/deploy/presidio/values.yaml b/tools/deploy/presidio/values.yaml index 8640458..3dae024 100644 --- a/tools/deploy/presidio/values.yaml +++ b/tools/deploy/presidio/values.yaml @@ -1,4 +1,4 @@ -registry: mcr.microsoft.com +registry: ghcr.io # Image pull secret # privateRegistry: acr-auth From d212e532fe71c2c4663298a50a1bbbea2ac08580 Mon Sep 17 00:00:00 2001 From: Laure Vancau Date: Thu, 23 Oct 2025 13:10:05 +0200 Subject: [PATCH 02/17] test: ci --- .github/workflows/build-images-manifests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-images-manifests.yml b/.github/workflows/build-images-manifests.yml index 319cdd9..d3b3183 100644 --- a/.github/workflows/build-images-manifests.yml +++ b/.github/workflows/build-images-manifests.yml @@ -4,7 +4,7 @@ on: pull_request: branches: [ main ] push: - branches: [ main ] + # branches: [ main ] workflow_dispatch: concurrency: From f9076710c272e53e2d7742f4811b098abab01902 Mon Sep 17 00:00:00 2001 From: Laure Vancau Date: Thu, 23 Oct 2025 13:11:25 +0200 Subject: [PATCH 03/17] test:ci --- .github/workflows/build-images-manifests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-images-manifests.yml b/.github/workflows/build-images-manifests.yml index d3b3183..dfca0f2 100644 --- a/.github/workflows/build-images-manifests.yml +++ b/.github/workflows/build-images-manifests.yml @@ -4,7 +4,8 @@ on: pull_request: branches: [ main ] push: - # branches: [ main ] + branches: [ '**' ] + #branches: [ main ] workflow_dispatch: concurrency: From d4e14600dfd6e740c3a83c19dc6ffd3f0e350b06 Mon Sep 17 00:00:00 2001 From: Laure Vancau Date: Thu, 23 Oct 2025 13:14:24 +0200 Subject: [PATCH 04/17] test:ci --- .github/workflows/build-images-manifests.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build-images-manifests.yml b/.github/workflows/build-images-manifests.yml index dfca0f2..a3fc231 100644 --- a/.github/workflows/build-images-manifests.yml +++ b/.github/workflows/build-images-manifests.yml @@ -1,11 +1,9 @@ name: Presidio Docker Build on: + push: pull_request: branches: [ main ] - push: - branches: [ '**' ] - #branches: [ main ] workflow_dispatch: concurrency: From 002a61f6d4dc4df712a699c40f67bcf206a94151 Mon Sep 17 00:00:00 2001 From: Laure Vancau Date: Thu, 23 Oct 2025 13:15:51 +0200 Subject: [PATCH 05/17] test:ci --- .github/workflows/build-images-manifests.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-images-manifests.yml b/.github/workflows/build-images-manifests.yml index a3fc231..1ccc07b 100644 --- a/.github/workflows/build-images-manifests.yml +++ b/.github/workflows/build-images-manifests.yml @@ -2,13 +2,13 @@ name: Presidio Docker Build on: push: - pull_request: - branches: [ main ] - workflow_dispatch: + # pull_request: + # branches: [ main ] + # workflow_dispatch: -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true +# concurrency: +# group: ${{ github.workflow }}-${{ github.ref }} +# cancel-in-progress: true permissions: read-all From 0ae6d1c138c7210ee32860ad69928d74ad681ed4 Mon Sep 17 00:00:00 2001 From: Laure Vancau Date: Thu, 23 Oct 2025 13:18:19 +0200 Subject: [PATCH 06/17] fix: if condition in action --- .github/workflows/build-images-manifests.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-images-manifests.yml b/.github/workflows/build-images-manifests.yml index 1ccc07b..56ff644 100644 --- a/.github/workflows/build-images-manifests.yml +++ b/.github/workflows/build-images-manifests.yml @@ -2,9 +2,9 @@ name: Presidio Docker Build on: push: - # pull_request: + # pull_request: # Future run on main pushes only. # branches: [ main ] - # workflow_dispatch: + workflow_dispatch: # concurrency: # group: ${{ github.workflow }}-${{ github.ref }} @@ -20,7 +20,6 @@ jobs: build-platform-images: name: Build ${{ matrix.image }} (${{ matrix.platform }}) runs-on: ${{ matrix.runner }} - if: github.ref == 'refs/heads/main' permissions: id-token: write contents: read @@ -91,7 +90,6 @@ jobs: name: Create Multi-Platform Manifests runs-on: ubuntu-latest needs: build-platform-images - if: github.ref == 'refs/heads/main' permissions: id-token: write contents: read From d4488ab3833d78dd1ad8fdc9b96b26f44741cd5e Mon Sep 17 00:00:00 2001 From: Laure Vancau Date: Thu, 23 Oct 2025 13:20:49 +0200 Subject: [PATCH 07/17] fix: registry env variable --- .github/workflows/build-images-manifests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-images-manifests.yml b/.github/workflows/build-images-manifests.yml index 56ff644..0dbe29d 100644 --- a/.github/workflows/build-images-manifests.yml +++ b/.github/workflows/build-images-manifests.yml @@ -67,7 +67,7 @@ jobs: - name: Log in to the Container registry uses: docker/login-action@v3.0.0 with: - registry: ${{ env.REGISTRY }} + registry: ${{ env.REGISTRY_NAME }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} From 69ab33d2a0f5c5853b854f92bb8439216d72330e Mon Sep 17 00:00:00 2001 From: Laure Vancau Date: Thu, 23 Oct 2025 13:37:31 +0200 Subject: [PATCH 08/17] fix: paths for push --- .github/workflows/build-images-manifests.yml | 23 ++++++++------------ 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-images-manifests.yml b/.github/workflows/build-images-manifests.yml index 0dbe29d..1102190 100644 --- a/.github/workflows/build-images-manifests.yml +++ b/.github/workflows/build-images-manifests.yml @@ -13,7 +13,8 @@ on: permissions: read-all env: - REGISTRY_NAME: ghcr.io # SDSC ADD-ON + REGISTRY_NAME: ghcr.io # SDSC ADD-ON + USERNAME: ${{ github.repository_owner }} TAG: gha${{ github.run_number }} jobs: @@ -78,13 +79,10 @@ jobs: docker buildx build \ --platform ${{ matrix.platform }} \ --push \ - --tag ${{ env.REGISTRY_NAME }}/${{ matrix.image }}:${{ env.TAG }}-${PLATFORM_TAG} \ - --cache-from type=registry,ref=${{ env.REGISTRY_NAME }}/public/${{ matrix.image }}:latest \ + --tag ${{ env.REGISTRY_NAME }}/${{ env.USERNAME }}/${{ matrix.image }}:${{ env.TAG }}-${PLATFORM_TAG} \ + --cache-from type=registry,ref=${{ env.REGISTRY_NAME }}/${{ env.USERNAME }}/${{ matrix.image }}:latest \ --cache-to type=inline \ ./${{ matrix.image }} - env: - REGISTRY_NAME: ${{ env.REGISTRY_NAME }} - TAG: ${{ env.TAG }} create-manifests: name: Create Multi-Platform Manifests @@ -99,7 +97,7 @@ jobs: - name: Log in to the Container registry uses: docker/login-action@v3.0.0 with: - registry: ${{ env.REGISTRY }} + registry: ${{ env.REGISTRY_NAME }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} @@ -113,10 +111,7 @@ jobs: for image in "${IMAGES[@]}"; do echo "Creating manifest for $image" docker buildx imagetools create \ - --tag ${{ env.REGISTRY_NAME }}/${image}:${{ env.TAG }} \ - ${{ env.REGISTRY_NAME }}/${image}:${{ env.TAG }}-linux-amd64 \ - ${{ env.REGISTRY_NAME }}/${image}:${{ env.TAG }}-linux-arm64 - done - env: - REGISTRY_NAME: ${{ env.REGISTRY_NAME }} - TAG: ${{ env.TAG }} \ No newline at end of file + --tag ${{ env.REGISTRY_NAME }}/${{ env.USERNAME }}/${image}:${{ env.TAG }} \ + ${{ env.REGISTRY_NAME }}/${{ env.USERNAME }}/${image}:${{ env.TAG }}-linux-amd64 \ + ${{ env.REGISTRY_NAME }}/${{ env.USERNAME }}/${image}:${{ env.TAG }}-linux-arm64 + done \ No newline at end of file From 5fd667a1204439d59aadc99e8afffebc0d75ee65 Mon Sep 17 00:00:00 2001 From: Laure Vancau Date: Thu, 23 Oct 2025 13:48:01 +0200 Subject: [PATCH 09/17] test: actor instead of repository owner --- .github/workflows/build-images-manifests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-images-manifests.yml b/.github/workflows/build-images-manifests.yml index 1102190..eeeae2a 100644 --- a/.github/workflows/build-images-manifests.yml +++ b/.github/workflows/build-images-manifests.yml @@ -14,7 +14,7 @@ permissions: read-all env: REGISTRY_NAME: ghcr.io # SDSC ADD-ON - USERNAME: ${{ github.repository_owner }} + USERNAME: ${{ github.actor }} TAG: gha${{ github.run_number }} jobs: From 1c74f99838f1254bc52f3b984c5ee9a07c7e94ff Mon Sep 17 00:00:00 2001 From: Laure Vancau Date: Thu, 23 Oct 2025 16:50:51 +0200 Subject: [PATCH 10/17] fix: change permissions --- .github/workflows/build-images-manifests.yml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/.github/workflows/build-images-manifests.yml b/.github/workflows/build-images-manifests.yml index eeeae2a..3763c4f 100644 --- a/.github/workflows/build-images-manifests.yml +++ b/.github/workflows/build-images-manifests.yml @@ -6,15 +6,9 @@ on: # branches: [ main ] workflow_dispatch: -# concurrency: -# group: ${{ github.workflow }}-${{ github.ref }} -# cancel-in-progress: true - -permissions: read-all - env: REGISTRY_NAME: ghcr.io # SDSC ADD-ON - USERNAME: ${{ github.actor }} + USERNAME: ${{ github.repository_owner }} TAG: gha${{ github.run_number }} jobs: @@ -36,15 +30,6 @@ jobs: - image: presidio-image-redactor platform: linux/amd64 runner: ubuntu-latest - - image: presidio-anonymizer - platform: linux/arm64 - runner: ubuntu-24.04-arm - - image: presidio-analyzer - platform: linux/arm64 - runner: ubuntu-24.04-arm - - image: presidio-image-redactor - platform: linux/arm64 - runner: ubuntu-24.04-arm steps: # SDSC ADD-ON - name: Get latest Presidio release tag From 29e58ed213658f712c1c50008a8e10e89597b644 Mon Sep 17 00:00:00 2001 From: Laure Vancau Date: Thu, 23 Oct 2025 16:53:09 +0200 Subject: [PATCH 11/17] fix: remove all permissions specifications --- .github/workflows/build-images-manifests.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/build-images-manifests.yml b/.github/workflows/build-images-manifests.yml index 3763c4f..0cb1ee5 100644 --- a/.github/workflows/build-images-manifests.yml +++ b/.github/workflows/build-images-manifests.yml @@ -15,9 +15,6 @@ jobs: build-platform-images: name: Build ${{ matrix.image }} (${{ matrix.platform }}) runs-on: ${{ matrix.runner }} - permissions: - id-token: write - contents: read strategy: matrix: include: @@ -73,9 +70,6 @@ jobs: name: Create Multi-Platform Manifests runs-on: ubuntu-latest needs: build-platform-images - permissions: - id-token: write - contents: read steps: # SDSC ADD-ON # https://github.com/docker/login-action From 2792f3e9c8c1410c30b5f2769fe09a211f55547a Mon Sep 17 00:00:00 2001 From: Laure Vancau Date: Thu, 23 Oct 2025 17:00:58 +0200 Subject: [PATCH 12/17] test: manifest rendering --- .github/workflows/build-images-manifests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-images-manifests.yml b/.github/workflows/build-images-manifests.yml index 0cb1ee5..b2ef730 100644 --- a/.github/workflows/build-images-manifests.yml +++ b/.github/workflows/build-images-manifests.yml @@ -93,4 +93,6 @@ jobs: --tag ${{ env.REGISTRY_NAME }}/${{ env.USERNAME }}/${image}:${{ env.TAG }} \ ${{ env.REGISTRY_NAME }}/${{ env.USERNAME }}/${image}:${{ env.TAG }}-linux-amd64 \ ${{ env.REGISTRY_NAME }}/${{ env.USERNAME }}/${image}:${{ env.TAG }}-linux-arm64 - done \ No newline at end of file + done + env: + TAG: latest From 882fc11663f382525d913213ac807bdd68e14a82 Mon Sep 17 00:00:00 2001 From: Laure Vancau Date: Thu, 23 Oct 2025 17:06:13 +0200 Subject: [PATCH 13/17] fix: remove arm version in manifests --- .github/workflows/build-images-manifests.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build-images-manifests.yml b/.github/workflows/build-images-manifests.yml index b2ef730..114a178 100644 --- a/.github/workflows/build-images-manifests.yml +++ b/.github/workflows/build-images-manifests.yml @@ -91,8 +91,5 @@ jobs: echo "Creating manifest for $image" docker buildx imagetools create \ --tag ${{ env.REGISTRY_NAME }}/${{ env.USERNAME }}/${image}:${{ env.TAG }} \ - ${{ env.REGISTRY_NAME }}/${{ env.USERNAME }}/${image}:${{ env.TAG }}-linux-amd64 \ - ${{ env.REGISTRY_NAME }}/${{ env.USERNAME }}/${image}:${{ env.TAG }}-linux-arm64 + ${{ env.REGISTRY_NAME }}/${{ env.USERNAME }}/${image}:${{ env.TAG }}-linux-amd64 done - env: - TAG: latest From 7fee42c0f0d3d9e6f6b1585f02cff0f668692474 Mon Sep 17 00:00:00 2001 From: Laure Vancau Date: Fri, 24 Oct 2025 09:45:19 +0200 Subject: [PATCH 14/17] feat: vendir and kustomization setup --- .github/workflows/build-images-manifests.yml | 7 +- .gitignore | 6 ++ docs/services.md | 88 +++++++++++++++++++ external/vendir.lock.yml | 11 +++ external/vendir.yaml | 10 +++ src/kustomization.yaml | 4 + src/presidio/kustomization.yaml | 11 +++ .../presidio/ytt/out}/values.yaml | 1 - tools/deploy/presidio/.helmignore | 21 ----- tools/deploy/presidio/Chart.yaml | 5 -- tools/deploy/presidio/templates/_helpers.tpl | 40 --------- .../templates/analyzer-deployment.yaml | 38 -------- .../presidio/templates/analyzer-service.yaml | 20 ----- .../templates/anonymizer-deployment.yaml | 38 -------- .../templates/anonymizer-service.yaml | 20 ----- tools/deploy/presidio/templates/ingress.yaml | 36 -------- 16 files changed, 134 insertions(+), 222 deletions(-) create mode 100644 docs/services.md create mode 100644 external/vendir.lock.yml create mode 100644 external/vendir.yaml create mode 100644 src/kustomization.yaml create mode 100644 src/presidio/kustomization.yaml rename {tools/deploy/presidio => src/presidio/ytt/out}/values.yaml (97%) delete mode 100644 tools/deploy/presidio/.helmignore delete mode 100644 tools/deploy/presidio/Chart.yaml delete mode 100644 tools/deploy/presidio/templates/_helpers.tpl delete mode 100644 tools/deploy/presidio/templates/analyzer-deployment.yaml delete mode 100644 tools/deploy/presidio/templates/analyzer-service.yaml delete mode 100644 tools/deploy/presidio/templates/anonymizer-deployment.yaml delete mode 100644 tools/deploy/presidio/templates/anonymizer-service.yaml delete mode 100644 tools/deploy/presidio/templates/ingress.yaml diff --git a/.github/workflows/build-images-manifests.yml b/.github/workflows/build-images-manifests.yml index 114a178..a5eeb5b 100644 --- a/.github/workflows/build-images-manifests.yml +++ b/.github/workflows/build-images-manifests.yml @@ -24,9 +24,10 @@ jobs: - image: presidio-analyzer platform: linux/amd64 runner: ubuntu-latest - - image: presidio-image-redactor - platform: linux/amd64 - runner: ubuntu-latest + # Note: do we want this part of presidio ? Maybe future feature ? + # - image: presidio-image-redactor + # platform: linux/amd64 + # runner: ubuntu-latest steps: # SDSC ADD-ON - name: Get latest Presidio release tag diff --git a/.gitignore b/.gitignore index 5add120..8f28196 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,10 @@ .direnv/ + +# third party manifests +external/helm/* +external/ytt/* +external/.vendir* + # Byte-compiled / optimized / DLL files __pycache__/ *.py[codz] diff --git a/docs/services.md b/docs/services.md new file mode 100644 index 0000000..1c349cc --- /dev/null +++ b/docs/services.md @@ -0,0 +1,88 @@ +# Services management + +The deployment defines multiple service (or application), each being a +collection of kubernetes manifests located in `src//`. + +## Structure + +- `external/`: third party resources +- `src/`: deployable manifests +- secrets are encrypted with sops+age and persisted in `src/secrets/` + +Each service is structured as follows (supported tools are `ytt` and `helm`): + +```text +├── external +│ └── +│ └── /... # <- third party templates +└── src + └── + ├── additional-manifest.yaml # <- custom manifests for this deployment + ├── kustomization.yaml # <- kustomization file to select resources + └── + ├── out/... # <- rendered manifests + └── values.yaml # <- values used for templating +``` + +## Templating + +[ytt](https://carvel.dev/ytt) is the preferred rendering engine, but helm is +also supported as many upstream templates are distributed with +[helm](https://helm.sh). + +When running `just render`, we attempt to render each service with helm and then +with ytt and save the rendered manifests in the repository. + +## Deployment + +When deploying with `just deploy`, deployment is done with kustomize +(`kubectl -k`). This means that the `src` and each of its subdirectories contain +a `kustomization.yaml` file which determine what manifests are included in the +deployment. + +For example, running `just deploy src/` will recursively parse +`src/kustomization.yaml` and the `kustomization.yaml` from each resources +declared in that file. This allows to simply exclude services or manifests by +commenting them out of `kustomization.yaml`. + +## Updating a service + +Here is the typical workflow to re-deploy a service that has been updated +upstream. + +1. Update the external manifest templates. This will update the `vendir` lock + file and fetch the latest templates into `external//`. + +```bash +just external::refresh +``` + +2. Render the manifests with the new templates. + +```bash +just render +``` + +> [!NOTE] +> This may fail if the new templates broke compatibility with existing values, +> in which case you will need to update your values in +> `src///values.yaml`. Also watch out in case the upstream added +> new template files, as you may need to include them in the service +> `kustomization.yaml`. + +3. Deploy the updated manifests. + +```bash +just deploy src/ +``` + +> [!IMPORTANT] +> In some cases, you may want to manually delete resources related to the +> service. You can achieve that with `just delete src/` or use +> `kubectl delete` to delete specific resoruces. + +## Adding custom manifests + +Custom manifests (e.g. additional volumes) can be added inside `src//`, +but they need to be added as a resource in `kustomization.yaml` file in the same +directory. diff --git a/external/vendir.lock.yml b/external/vendir.lock.yml new file mode 100644 index 0000000..6db55bf --- /dev/null +++ b/external/vendir.lock.yml @@ -0,0 +1,11 @@ +apiVersion: vendir.k14s.io/v1alpha1 +directories: +- contents: + - git: + commitTitle: Add label to external PRs (#1707)... + sha: af1c524460ad62e17313520a3cbb618b062b75cb + tags: + - 2.2.360 + path: . + path: ytt/presidio +kind: LockConfig diff --git a/external/vendir.yaml b/external/vendir.yaml new file mode 100644 index 0000000..c6f21a6 --- /dev/null +++ b/external/vendir.yaml @@ -0,0 +1,10 @@ +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: + - path: ytt/presidio + contents: + - path: . + git: + url: https://github.com/microsoft/presidio + ref: refs/tags/2.2.360 + newRootPath: docs/samples/deployments/k8s/charts/presidio diff --git a/src/kustomization.yaml b/src/kustomization.yaml new file mode 100644 index 0000000..158097d --- /dev/null +++ b/src/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./presidio diff --git a/src/presidio/kustomization.yaml b/src/presidio/kustomization.yaml new file mode 100644 index 0000000..bb343ce --- /dev/null +++ b/src/presidio/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./namespaces.yaml + - ./presidio-pvc.yaml + - ./helm/out/presidio/templates/analyzer-deployment.yaml + - ./helm/out/presidio/templates/anonymizer-deployment.yaml + - ./helm/out/presidio/templates/analyzer-service.yaml + - ./helm/out/presidio/templates/anonymizer-service.yaml + - ./helm/out/presidio/templates/analyzer-ingress.yaml +namespace: presidio \ No newline at end of file diff --git a/tools/deploy/presidio/values.yaml b/src/presidio/ytt/out/values.yaml similarity index 97% rename from tools/deploy/presidio/values.yaml rename to src/presidio/ytt/out/values.yaml index 3dae024..22f976d 100644 --- a/tools/deploy/presidio/values.yaml +++ b/src/presidio/ytt/out/values.yaml @@ -1,7 +1,6 @@ registry: ghcr.io # Image pull secret -# privateRegistry: acr-auth tag: latest # supported types are nginx (wip: traefik and istio) diff --git a/tools/deploy/presidio/.helmignore b/tools/deploy/presidio/.helmignore deleted file mode 100644 index f0c1319..0000000 --- a/tools/deploy/presidio/.helmignore +++ /dev/null @@ -1,21 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*~ -# Various IDEs -.project -.idea/ -*.tmproj diff --git a/tools/deploy/presidio/Chart.yaml b/tools/deploy/presidio/Chart.yaml deleted file mode 100644 index aa69be3..0000000 --- a/tools/deploy/presidio/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v2 -description: A context aware, born to the cloud, customizable data loss prevention service -name: presidio -version: 2.0 -appVersion: latest diff --git a/tools/deploy/presidio/templates/_helpers.tpl b/tools/deploy/presidio/templates/_helpers.tpl deleted file mode 100644 index 935b0ad..0000000 --- a/tools/deploy/presidio/templates/_helpers.tpl +++ /dev/null @@ -1,40 +0,0 @@ -{{/* vim: set filetype=mustache */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "presidio.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "presidio.fullname" -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{- define "presidio.analyzer.fullname" -}} -{{ include "presidio.fullname" . | printf "%s-analyzer" }} -{{- end -}} -{{- define "presidio.anonymizer.fullname" -}} -{{ include "presidio.fullname" . | printf "%s-anonymizer" }} -{{- end -}} -{{- define "presidio.anonymizerimage.fullname" -}} -{{ include "presidio.fullname" . | printf "%s-image-redactor" }} -{{- end -}} - -{{- define "presidio.analyzer.address" -}} -{{template "presidio.analyzer.fullname" .}}:{{.Values.analyzer.service.externalPort}} -{{- end -}} - -{{- define "presidio.anonymizer.address" -}} -{{template "presidio.anonymizer.fullname" .}}:{{.Values.anonymizer.service.externalPort}} -{{- end -}} - -{{- define "presidio.anonymizerimage.address" -}} -{{template "presidio.anonymizerimage.fullname" .}}:{{.Values.anonymizerimage.service.externalPort}} -{{- end -}} - -{{- define "presidio.rbac.version" }}rbac.authorization.k8s.io/v1{{ end -}} \ No newline at end of file diff --git a/tools/deploy/presidio/templates/analyzer-deployment.yaml b/tools/deploy/presidio/templates/analyzer-deployment.yaml deleted file mode 100644 index 1e693d8..0000000 --- a/tools/deploy/presidio/templates/analyzer-deployment.yaml +++ /dev/null @@ -1,38 +0,0 @@ -{{ $fullname := include "presidio.analyzer.fullname" . }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ $fullname }} - labels: - app: {{ $fullname }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" -spec: - replicas: {{ .Values.analyzer.replicas }} - selector: - matchLabels: - app: {{ $fullname }} - template: - metadata: - labels: - app: {{ $fullname }} - spec: - containers: - - name: {{ .Chart.Name }} - image: "{{ .Values.registry }}/{{ .Values.analyzer.name }}:{{ default .Chart.AppVersion .Values.tag }}" - imagePullPolicy: {{ default "IfNotPresent" .Values.analyzer.imagePullPolicy }} - ports: - - containerPort: {{ .Values.analyzer.service.internalPort }} - resources: - requests: - memory: {{ .Values.analyzer.container.resources.requests.memory }} - cpu: {{ .Values.analyzer.container.resources.requests.cpu }} - limits: - memory: {{ .Values.analyzer.container.resources.limits.memory }} - cpu: {{ .Values.analyzer.container.resources.limits.cpu }} - env: - - name: PORT - value: {{ .Values.analyzer.service.internalPort | quote }} - {{ if .Values.privateRegistry }}imagePullSecrets: - - name: {{.Values.privateRegistry}}{{ end }} \ No newline at end of file diff --git a/tools/deploy/presidio/templates/analyzer-service.yaml b/tools/deploy/presidio/templates/analyzer-service.yaml deleted file mode 100644 index 5a2f693..0000000 --- a/tools/deploy/presidio/templates/analyzer-service.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{ $fullname := include "presidio.analyzer.fullname" . }} -apiVersion: v1 -kind: Service -metadata: - name: {{ $fullname }} - labels: - app: {{ $fullname }} - service: {{ $fullname }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" -spec: - type: {{ .Values.analyzer.service.type }} - ports: - - port: {{ .Values.analyzer.service.externalPort }} - targetPort: {{ .Values.analyzer.service.internalPort }} - protocol: TCP - name: {{ .Values.analyzer.service.name }} - selector: - app: {{ $fullname }} \ No newline at end of file diff --git a/tools/deploy/presidio/templates/anonymizer-deployment.yaml b/tools/deploy/presidio/templates/anonymizer-deployment.yaml deleted file mode 100644 index f211ca0..0000000 --- a/tools/deploy/presidio/templates/anonymizer-deployment.yaml +++ /dev/null @@ -1,38 +0,0 @@ -{{ $fullname := include "presidio.anonymizer.fullname" . }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ $fullname }} - labels: - app: {{ $fullname }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" -spec: - replicas: {{ .Values.anonymizer.replicas }} - selector: - matchLabels: - app: {{ $fullname }} - template: - metadata: - labels: - app: {{ $fullname }} - spec: - containers: - - name: {{ .Chart.Name }} - image: "{{ .Values.registry }}/{{ .Values.anonymizer.name }}:{{ default .Chart.AppVersion .Values.tag }}" - imagePullPolicy: {{ default "IfNotPresent" .Values.anonymizer.imagePullPolicy }} - ports: - - containerPort: {{ .Values.anonymizer.service.internalPort }} - resources: - requests: - memory: {{ .Values.anonymizer.container.resources.requests.memory }} - cpu: {{ .Values.anonymizer.container.resources.requests.cpu }} - limits: - memory: {{ .Values.anonymizer.container.resources.limits.memory }} - cpu: {{ .Values.anonymizer.container.resources.limits.cpu }} - env: - - name: PORT - value: {{ .Values.anonymizer.service.internalPort | quote }} - {{ if .Values.privateRegistry }}imagePullSecrets: - - name: {{.Values.privateRegistry}}{{ end }} \ No newline at end of file diff --git a/tools/deploy/presidio/templates/anonymizer-service.yaml b/tools/deploy/presidio/templates/anonymizer-service.yaml deleted file mode 100644 index 1cd4668..0000000 --- a/tools/deploy/presidio/templates/anonymizer-service.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{ $fullname := include "presidio.anonymizer.fullname" . }} -apiVersion: v1 -kind: Service -metadata: - name: {{ $fullname }} - labels: - app: {{ $fullname }} - service: {{ $fullname }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" -spec: - type: {{ .Values.anonymizer.service.type }} - ports: - - port: {{ .Values.anonymizer.service.externalPort }} - targetPort: {{ .Values.anonymizer.service.internalPort }} - protocol: TCP - name: {{ .Values.anonymizer.service.name }} - selector: - app: {{ $fullname }} \ No newline at end of file diff --git a/tools/deploy/presidio/templates/ingress.yaml b/tools/deploy/presidio/templates/ingress.yaml deleted file mode 100644 index 32976d7..0000000 --- a/tools/deploy/presidio/templates/ingress.yaml +++ /dev/null @@ -1,36 +0,0 @@ -{{- if and (.Values.ingress.enabled) (eq .Values.ingress.class "nginx") -}} -{{- $analyzerfullname := include "presidio.analyzer.fullname" . -}} -{{- $anonymizerfullname := include "presidio.anonymizer.fullname" . -}} -{{- $anonymizerimagefullname := include "presidio.anonymizerimage.fullname" . -}} -apiVersion: networking.k8s.io/v1beta1 -kind: Ingress -metadata: - name: presidio-ingress - labels: - app: presidio-ingress - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} - annotations: - kubernetes.io/ingress.class: {{ .Values.ingress.class }} - nginx.ingress.kubernetes.io/rewrite-target: "/$2" -spec: - rules: - - http: - paths: - - backend: - serviceName: {{ $analyzerfullname }} - servicePort: {{ .Values.analyzer.service.externalPort }} - path: /{{ $analyzerfullname }}(/|$)(.*) - pathType: Prefix - - backend: - serviceName: {{ $anonymizerfullname }} - servicePort: {{ .Values.anonymizer.service.externalPort }} - path: /{{ $anonymizerfullname }}(/|$)(.*) - pathType: Prefix - - backend: - serviceName: {{ $anonymizerimagefullname }} - servicePort: {{ .Values.anonymizerimage.service.externalPort }} - path: /{{ $anonymizerimagefullname }}(/|$)(.*) - pathType: Prefix -{{- end -}} From 482bb5bd1f1e4aec1cd083cd055c9b66b80968f9 Mon Sep 17 00:00:00 2001 From: Laure Vancau Date: Fri, 24 Oct 2025 10:59:15 +0200 Subject: [PATCH 15/17] feat: functional render pipeline ytt and helm --- .github/workflows/build-images-manifests.yml | 8 ++-- docs/presidio-poc.md | 37 +++++++++++-------- external/vendir.yaml | 10 +++++ justfile | 9 ++++- .../templates/analyzer-deployment.yaml | 37 +++++++++++++++++++ .../presidio/templates/analyzer-service.yaml | 21 +++++++++++ .../templates/anonymizer-deployment.yaml | 37 +++++++++++++++++++ .../anonymizer-image-deployment.yaml | 37 +++++++++++++++++++ .../templates/anonymizer-image-service.yaml | 21 +++++++++++ .../templates/anonymizer-service.yaml | 21 +++++++++++ src/presidio/{ytt/out => helm}/values.yaml | 0 src/presidio/kustomization.yaml | 2 +- 12 files changed, 219 insertions(+), 21 deletions(-) create mode 100644 src/presidio/helm/out/presidio/templates/analyzer-deployment.yaml create mode 100644 src/presidio/helm/out/presidio/templates/analyzer-service.yaml create mode 100644 src/presidio/helm/out/presidio/templates/anonymizer-deployment.yaml create mode 100644 src/presidio/helm/out/presidio/templates/anonymizer-image-deployment.yaml create mode 100644 src/presidio/helm/out/presidio/templates/anonymizer-image-service.yaml create mode 100644 src/presidio/helm/out/presidio/templates/anonymizer-service.yaml rename src/presidio/{ytt/out => helm}/values.yaml (100%) diff --git a/.github/workflows/build-images-manifests.yml b/.github/workflows/build-images-manifests.yml index a5eeb5b..8af44a6 100644 --- a/.github/workflows/build-images-manifests.yml +++ b/.github/workflows/build-images-manifests.yml @@ -1,8 +1,8 @@ name: Presidio Docker Build on: - push: - # pull_request: # Future run on main pushes only. + push: + # pull_request: # Future run on main pushes only. # branches: [ main ] workflow_dispatch: @@ -35,7 +35,7 @@ jobs: run: | tag=$(curl -s https://api.github.com/repos/microsoft/presidio/releases/latest | jq -r .tag_name) echo "tag=$tag" >> $GITHUB_OUTPUT - + # SDSC ADD-ON - name: Checkout Presidio (latest release) uses: actions/checkout@v5 @@ -87,7 +87,7 @@ jobs: - name: Create all multi-platform manifests run: | IMAGES=("presidio-anonymizer" "presidio-analyzer" "presidio-image-redactor") - + for image in "${IMAGES[@]}"; do echo "Creating manifest for $image" docker buildx imagetools create \ diff --git a/docs/presidio-poc.md b/docs/presidio-poc.md index 51c8e86..8713cba 100644 --- a/docs/presidio-poc.md +++ b/docs/presidio-poc.md @@ -8,14 +8,17 @@ - can be deployed as an API server using a compose stack ## API usage + 2-steps: + - analyze: NER from raw text using models - anonymize: config (rule) based processing of pre-detected PII ### analyze + - Minimal requirements: text + language. By default, all recognizers for that language are enabled. ```sh - $ curl http://localhost:5002/analyze -s --header "Content-Type: application/json" --request POST --data '{"text": "John Smith drivers license is AC432223","language": "en"}' | jq + $ curl http://localhost:5002/analyze -s --header "Content-Type: application/json" --request POST --data '{"text": "John Smith drivers license is AC432223","language": "en"}' | jq [ { "analysis_explanation": null, @@ -33,11 +36,12 @@ } ] ``` -- analysis can be controlled by setting detection score, selecting entities, adding context words and adding a correlation id(?) +- analysis can be controlled by setting detection score, selecting entities, adding context words and adding a correlation id(?) - ad-hoc pattern (regex) recognizers can be provided as json objects - a correlation-id (hash) can be given to append to logs for easier grouping of analyses in logs / traces. ### anonymize + - By default, the anonymization replaces all detected identifies by their type (e.g. ) in the input text. - An anonymizer dictionary can be provided to associate specific anonymization procedure to specific entity types. - Two inputs must be given to the endpoint: @@ -45,7 +49,9 @@ - the response from the analyze step (detected entities and their positions) ### artificial sample + Input: + ``` Prof. Gérard Waeber, Chef de service Tél: +41 21 314 68 85 / Fax: +41 21 314 08 95 @@ -77,8 +83,10 @@ jfldéijf Dr Médecin 00 Formateur Chef de clinique ``` + - ## initial tests -Works with example artifical lettre de sortie. + Works with example artifical lettre de sortie. + ```python import json import requests @@ -129,7 +137,9 @@ print( ## limitations ### potential improvements + Model configuration + ```yaml # config.yaml nlp_engine_name: spacy @@ -157,30 +167,28 @@ ner_model_configuration: ``` Recognizer configuration + ```yaml # recognizers.yaml recognizers: - - - name: "Swiss Zip code Recognizer" + - name: "Swiss Zip code Recognizer" supported_languages: - language: fr context: [adresse, postal] - language: de - context: [ort,] + context: [ort] - language: it context: [...] patterns: - - - name: "zip code (weak)" - regex: "(\\b\\d{5}(?:\\-\\d{4})?\\b)" - score: 0.01 + - name: "zip code (weak)" + regex: "(\\b\\d{5}(?:\\-\\d{4})?\\b)" + score: 0.01 context: - - zip - - code + - zip + - code supported_entity: "ZIP" - - - name: "Titles recognizer" + - name: "Titles recognizer" supported_language: "en" supported_entity: "TITLE" deny_list: @@ -190,5 +198,4 @@ recognizers: - Miss - Dr. - Prof. - ``` diff --git a/external/vendir.yaml b/external/vendir.yaml index c6f21a6..89c6020 100644 --- a/external/vendir.yaml +++ b/external/vendir.yaml @@ -8,3 +8,13 @@ directories: url: https://github.com/microsoft/presidio ref: refs/tags/2.2.360 newRootPath: docs/samples/deployments/k8s/charts/presidio + # - path: helm/presidio + # contents: + # - path: . + # helmChart: + # name: presidio + # version: 2.2.360 + # git: + # url: https://github.com/microsoft/presidio + # ref: refs/tags/2.2.360 + # subPath: docs/samples/deployments/k8s/charts/presidio diff --git a/justfile b/justfile index 6618ef1..5940e2c 100644 --- a/justfile +++ b/justfile @@ -33,11 +33,18 @@ render-ytt dir="src": fd '^ytt$' {{dir}} \ -x sh -c 'ytt -f {}/values.yaml -f external/ytt/$(basename {//}) --output-files {}/out' +# Render when the code was pulled in via ytt but is a helm template +[private] +render-ytt-extract-helm-template dir="src": + # render mixed ytt + helm templates with our values into src//mix/out + fd '^helm$' {{dir}} \ + -x sh -c 'helm template $(basename {//}) external/ytt/$(basename {//}) -f {}/values.yaml --output-dir {}/out' + # Render manifests render dir="src": just fetch && \ - just render-helm {{dir}} && \ just render-ytt {{dir}} && \ + just render-ytt-extract-helm-template {{dir}} && \ just format # Apply manifests in dir to the cluster. diff --git a/src/presidio/helm/out/presidio/templates/analyzer-deployment.yaml b/src/presidio/helm/out/presidio/templates/analyzer-deployment.yaml new file mode 100644 index 0000000..a5d3185 --- /dev/null +++ b/src/presidio/helm/out/presidio/templates/analyzer-deployment.yaml @@ -0,0 +1,37 @@ +--- +# Source: presidio/templates/analyzer-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: presidio-presidio-analyzer + labels: + app: presidio-presidio-analyzer + chart: "presidio-2" + release: "presidio" + heritage: "Helm" +spec: + replicas: 1 + selector: + matchLabels: + app: presidio-presidio-analyzer + template: + metadata: + labels: + app: presidio-presidio-analyzer + spec: + containers: + - name: presidio + image: "ghcr.io/presidio-analyzer:latest" + imagePullPolicy: Always + ports: + - containerPort: 8080 + resources: + requests: + memory: 1500Mi + cpu: 1500m + limits: + memory: 3000Mi + cpu: 2000m + env: + - name: PORT + value: "8080" diff --git a/src/presidio/helm/out/presidio/templates/analyzer-service.yaml b/src/presidio/helm/out/presidio/templates/analyzer-service.yaml new file mode 100644 index 0000000..ca2003d --- /dev/null +++ b/src/presidio/helm/out/presidio/templates/analyzer-service.yaml @@ -0,0 +1,21 @@ +--- +# Source: presidio/templates/analyzer-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: presidio-presidio-analyzer + labels: + app: presidio-presidio-analyzer + service: presidio-presidio-analyzer + chart: "presidio-2" + release: "presidio" + heritage: "Helm" +spec: + type: ClusterIP + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: presidio-presidio-analyzer diff --git a/src/presidio/helm/out/presidio/templates/anonymizer-deployment.yaml b/src/presidio/helm/out/presidio/templates/anonymizer-deployment.yaml new file mode 100644 index 0000000..597e5fd --- /dev/null +++ b/src/presidio/helm/out/presidio/templates/anonymizer-deployment.yaml @@ -0,0 +1,37 @@ +--- +# Source: presidio/templates/anonymizer-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: presidio-presidio-anonymizer + labels: + app: presidio-presidio-anonymizer + chart: "presidio-2" + release: "presidio" + heritage: "Helm" +spec: + replicas: 1 + selector: + matchLabels: + app: presidio-presidio-anonymizer + template: + metadata: + labels: + app: presidio-presidio-anonymizer + spec: + containers: + - name: presidio + image: "ghcr.io/presidio-anonymizer:latest" + imagePullPolicy: Always + ports: + - containerPort: 8080 + resources: + requests: + memory: 128Mi + cpu: 125m + limits: + memory: 512Mi + cpu: 500m + env: + - name: PORT + value: "8080" diff --git a/src/presidio/helm/out/presidio/templates/anonymizer-image-deployment.yaml b/src/presidio/helm/out/presidio/templates/anonymizer-image-deployment.yaml new file mode 100644 index 0000000..753e099 --- /dev/null +++ b/src/presidio/helm/out/presidio/templates/anonymizer-image-deployment.yaml @@ -0,0 +1,37 @@ +--- +# Source: presidio/templates/anonymizer-image-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: presidio-presidio-image-redactor + labels: + app: presidio-presidio-image-redactor + chart: "presidio-2" + release: "presidio" + heritage: "Helm" +spec: + replicas: 1 + selector: + matchLabels: + app: presidio-presidio-image-redactor + template: + metadata: + labels: + app: presidio-presidio-image-redactor + spec: + containers: + - name: presidio + image: "ghcr.io/presidio-image-redactor:latest" + imagePullPolicy: Always + ports: + - containerPort: 8080 + resources: + requests: + memory: 1500Mi + cpu: 1500m + limits: + memory: 3000Mi + cpu: 2000m + env: + - name: PORT + value: "8080" diff --git a/src/presidio/helm/out/presidio/templates/anonymizer-image-service.yaml b/src/presidio/helm/out/presidio/templates/anonymizer-image-service.yaml new file mode 100644 index 0000000..d7ac2d4 --- /dev/null +++ b/src/presidio/helm/out/presidio/templates/anonymizer-image-service.yaml @@ -0,0 +1,21 @@ +--- +# Source: presidio/templates/anonymizer-image-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: presidio-presidio-image-redactor + labels: + app: presidio-presidio-image-redactor + service: presidio-presidio-image-redactor + chart: "presidio-2" + release: "presidio" + heritage: "Helm" +spec: + type: ClusterIP + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: presidio-presidio-image-redactor diff --git a/src/presidio/helm/out/presidio/templates/anonymizer-service.yaml b/src/presidio/helm/out/presidio/templates/anonymizer-service.yaml new file mode 100644 index 0000000..d370b3d --- /dev/null +++ b/src/presidio/helm/out/presidio/templates/anonymizer-service.yaml @@ -0,0 +1,21 @@ +--- +# Source: presidio/templates/anonymizer-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: presidio-presidio-anonymizer + labels: + app: presidio-presidio-anonymizer + service: presidio-presidio-anonymizer + chart: "presidio-2" + release: "presidio" + heritage: "Helm" +spec: + type: ClusterIP + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: presidio-presidio-anonymizer diff --git a/src/presidio/ytt/out/values.yaml b/src/presidio/helm/values.yaml similarity index 100% rename from src/presidio/ytt/out/values.yaml rename to src/presidio/helm/values.yaml diff --git a/src/presidio/kustomization.yaml b/src/presidio/kustomization.yaml index bb343ce..2b7e178 100644 --- a/src/presidio/kustomization.yaml +++ b/src/presidio/kustomization.yaml @@ -8,4 +8,4 @@ resources: - ./helm/out/presidio/templates/analyzer-service.yaml - ./helm/out/presidio/templates/anonymizer-service.yaml - ./helm/out/presidio/templates/analyzer-ingress.yaml -namespace: presidio \ No newline at end of file +namespace: presidio From cc639e3a7ab6595c11fed1a46c6af49cc462c004 Mon Sep 17 00:00:00 2001 From: Laure Vancau Date: Fri, 24 Oct 2025 14:00:17 +0200 Subject: [PATCH 16/17] feat: patching in new configs --- .../presidio-analyzer/default-analyzer.yaml | 3 + .../default-recognizers.yaml | 201 ++++++++++++++++++ src/presidio/configmap.yaml | 8 + src/presidio/namespaces.yaml | 6 + src/presidio/patch-config.yaml | 21 ++ 5 files changed, 239 insertions(+) create mode 100644 src/presidio/conf/presidio-analyzer/default-analyzer.yaml create mode 100644 src/presidio/conf/presidio-analyzer/default-recognizers.yaml create mode 100644 src/presidio/configmap.yaml create mode 100644 src/presidio/namespaces.yaml create mode 100644 src/presidio/patch-config.yaml diff --git a/src/presidio/conf/presidio-analyzer/default-analyzer.yaml b/src/presidio/conf/presidio-analyzer/default-analyzer.yaml new file mode 100644 index 0000000..8cdc5e5 --- /dev/null +++ b/src/presidio/conf/presidio-analyzer/default-analyzer.yaml @@ -0,0 +1,3 @@ +supported_languages: + - en +default_score_threshold: 0 \ No newline at end of file diff --git a/src/presidio/conf/presidio-analyzer/default-recognizers.yaml b/src/presidio/conf/presidio-analyzer/default-recognizers.yaml new file mode 100644 index 0000000..0b3f02f --- /dev/null +++ b/src/presidio/conf/presidio-analyzer/default-recognizers.yaml @@ -0,0 +1,201 @@ +supported_languages: + - en +global_regex_flags: 26 + +recognizers: + # Recognizers listed here can either be loaded from the recognizers defined in code (type: predefined), + # or created based on the provided configuration (type: custom). + # For predefined: + # - If only a recognizer name is provided, a predefined recognizer with this name and default parameters will be loaded. + # - If a parameter isn't provided, the default one would be loaded. + # For custom: + # - See an example configuration here: https://github.com/microsoft/presidio/blob/main/presidio-analyzer/presidio_analyzer/conf/example_recognizers.yaml + # - Custom pattern recognizers with this configuration can be added to this file, with type: custom + # For recognizers supporting more than one language, an instance of the recognizer for each language will be created. + # For example, see the CreditCardRecognizer definition below: + - name: CreditCardRecognizer + supported_languages: + - language: en + context: [credit, card, visa, mastercard, cc, amex, discover, jcb, diners, maestro, instapayment] + - language: es + context: [tarjeta, credito, visa, mastercard, cc, amex, discover, jcb, diners, maestro, instapayment] + - language: it + - language: pl + type: predefined + + - name: UsBankRecognizer + supported_languages: + - en + type: predefined + + - name: UsLicenseRecognizer + supported_languages: + - en + type: predefined + + - name: UsItinRecognizer + supported_languages: + - en + type: predefined + + - name: UsPassportRecognizer + supported_languages: + - en + type: predefined + + - name: UsSsnRecognizer + supported_languages: + - en + type: predefined + + - name: NhsRecognizer + supported_languages: + - en + type: predefined + + - name: UkNinoRecognizer + supported_languages: + - en + type: predefined + enabled: false + + - name: SgFinRecognizer + supported_languages: + - en + type: predefined + enabled: false + + - name: AuAbnRecognizer + supported_languages: + - en + type: predefined + enabled: false + + - name: AuAcnRecognizer + supported_languages: + - en + type: predefined + enabled: false + + - name: AuTfnRecognizer + supported_languages: + - en + type: predefined + enabled: false + + - name: AuMedicareRecognizer + supported_languages: + - en + type: predefined + enabled: false + + - name: InPanRecognizer + supported_languages: + - en + type: predefined + enabled: false + + - name: InAadhaarRecognizer + supported_languages: + - en + type: predefined + enabled: false + + - name: InVehicleRegistrationRecognizer + supported_languages: + - en + type: predefined + enabled: false + + - name: InPassportRecognizer + supported_languages: + - en + type: predefined + enabled: false + + - name: EsNifRecognizer + supported_languages: + - es + type: predefined + + - name: EsNieRecognizer + supported_languages: + - es + type: predefined + + - name: ItDriverLicenseRecognizer + supported_languages: + - it + type: predefined + + - name: ItFiscalCodeRecognizer + supported_languages: + - it + type: predefined + + - name: ItVatCodeRecognizer + supported_languages: + - it + type: predefined + + - name: ItIdentityCardRecognizer + supported_languages: + - it + type: predefined + + - name: ItPassportRecognizer + supported_languages: + - it + type: predefined + + - name: PlPeselRecognizer + supported_languages: + - pl + type: predefined + + - name: KrRrnRecognizer + supported_languages: + - ko + - kr + type: predefined + enabled: false + + - name: ThTninRecognizer + supported_languages: + - th + type: predefined + enabled: false + + - name: CryptoRecognizer + type: predefined + + - name: DateRecognizer + type: predefined + + - name: EmailRecognizer + type: predefined + + - name: IbanRecognizer + type: predefined + + - name: IpRecognizer + type: predefined + + - name: MedicalLicenseRecognizer + type: predefined + + - name: PhoneRecognizer + type: predefined + + - name: UrlRecognizer + type: predefined + + - name: InVoterRecognizer + type: predefined + enabled: false + + - name: InGstinRecognizer + supported_languages: + - en + type: predefined + enabled: false \ No newline at end of file diff --git a/src/presidio/configmap.yaml b/src/presidio/configmap.yaml new file mode 100644 index 0000000..dd6478b --- /dev/null +++ b/src/presidio/configmap.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: presidio-analyzer-config +data: + default-analyzer.yaml: + |- + {{ .Files.Get "conf/presidio-analyzer/default-analyzer.yaml" | indent 2 }} diff --git a/src/presidio/namespaces.yaml b/src/presidio/namespaces.yaml new file mode 100644 index 0000000..9698b81 --- /dev/null +++ b/src/presidio/namespaces.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: presidio + labels: + name: presidio diff --git a/src/presidio/patch-config.yaml b/src/presidio/patch-config.yaml new file mode 100644 index 0000000..e35c7b0 --- /dev/null +++ b/src/presidio/patch-config.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: presidio-presidio-analyzer +spec: + selector: + matchLabels: + app: presidio-presidio-analyzer + template: + spec: + volumes: + - name: presidio-analyzer-config + configMap: + name: presidio-analyzer-config + containers: + - name: presidio + volumeMounts: + - name: presidio-analyzer-config + #TO-DO ensure this is the right path in the container + mountPath: presidio_analyzer/conf/default_analyzer.yaml + subPath: default-analyzer.yaml From 32def17ae6719b7696afbfeaf4bea5900aeb4ef6 Mon Sep 17 00:00:00 2001 From: Laure Vancau Date: Mon, 27 Oct 2025 09:42:36 +0100 Subject: [PATCH 17/17] fix: renaming of config maps for clarity and docker build only on main --- .github/workflows/build-images-manifests.yml | 3 +-- src/presidio/{configmap.yaml => configmap-analyzer.yaml} | 0 src/presidio/kustomization.yaml | 3 ++- src/presidio/{patch-config.yaml => patch-config-analyzer.yaml} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename src/presidio/{configmap.yaml => configmap-analyzer.yaml} (100%) rename src/presidio/{patch-config.yaml => patch-config-analyzer.yaml} (100%) diff --git a/.github/workflows/build-images-manifests.yml b/.github/workflows/build-images-manifests.yml index 8af44a6..4c23268 100644 --- a/.github/workflows/build-images-manifests.yml +++ b/.github/workflows/build-images-manifests.yml @@ -2,8 +2,7 @@ name: Presidio Docker Build on: push: - # pull_request: # Future run on main pushes only. - # branches: [ main ] + branches: [ main ] workflow_dispatch: env: diff --git a/src/presidio/configmap.yaml b/src/presidio/configmap-analyzer.yaml similarity index 100% rename from src/presidio/configmap.yaml rename to src/presidio/configmap-analyzer.yaml diff --git a/src/presidio/kustomization.yaml b/src/presidio/kustomization.yaml index 2b7e178..08098c8 100644 --- a/src/presidio/kustomization.yaml +++ b/src/presidio/kustomization.yaml @@ -2,10 +2,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ./namespaces.yaml - - ./presidio-pvc.yaml - ./helm/out/presidio/templates/analyzer-deployment.yaml - ./helm/out/presidio/templates/anonymizer-deployment.yaml - ./helm/out/presidio/templates/analyzer-service.yaml - ./helm/out/presidio/templates/anonymizer-service.yaml - ./helm/out/presidio/templates/analyzer-ingress.yaml + - ./configmap-analyzer.yaml + - ./patch-config-analyzer.yaml namespace: presidio diff --git a/src/presidio/patch-config.yaml b/src/presidio/patch-config-analyzer.yaml similarity index 100% rename from src/presidio/patch-config.yaml rename to src/presidio/patch-config-analyzer.yaml