Merge pull request #126 from chaen/doc_prod_deployment

Doc prod deployment

Author: chaen

.pre-commit-config.yaml

@@ -16,19 +16,19 @@ repos:
     - id: helm-docs
       name: helm-docs
       language: docker_image
-      entry: jnorwood/helm-docs:v1.11.2 --chart-search-root=diracx --output-file=../docs/admin/reference/values.md --template-files=../docs/admin/reference/values.md.gotmpl --chart-to-generate=diracx
+      entry: jnorwood/helm-docs:v1.14.2 --chart-search-root=diracx --output-file=../docs/admin/reference/values.md --template-files=../docs/admin/reference/values.md.gotmpl --chart-to-generate=diracx
       always_run: true
       pass_filenames: false
 
     - id: helm-lint
       name: helm-lint
       language: docker_image
-      entry: alpine/helm:3.11.1 lint diracx/ --set diracx.hostname=diracx.invalid
+      entry: alpine/helm:3.17.0 lint diracx/ --set diracx.hostname=diracx.invalid
       always_run: true
       pass_filenames: false
 
   - repo: https://github.com/koalaman/shellcheck-precommit
-    rev: v0.9.0
+    rev: v0.10.0
     hooks:
     - id: shellcheck
       # TODO: Render demo in CI and run the scripts through shell-check

README.md.gotmpl

@@ -169,39 +169,26 @@ helm diff upgrade diracx-demo  ./diracx --values .demo/values.yaml --set rabbitm
 
 # Actually run "helm upgrade" to apply changes
 helm upgrade diracx-demo ./diracx --values .demo/values.yaml
+
+# Retrieve the values.yaml that was used
+
+helm get values diracx-demo
 ```
 
 See [here](./docs/RUN_DEMO.md) for more details on what you can do to alter the behavior of the local installation.
 
 
 ## Deploying in production
 
+See [docs](./docs/RUN_DEMO.md)
 
-TODO: Link to k3s
-
-TODO: Explain how to download the values from helm
 
-TODO: add info about diracx-web
+### Deploying a custom branch
 
+This can be useful when debugging a problem, or running a specific version of the code which is not the one in the image.
 
-### Deploying a custom branch to DIRAC certification
+See [docs](./docs/DEBUGGING.md)
 
-Apply the following on top of the standard `values.yaml` file, replacing `USERNAME` and `BRANCH_NAME` with the appropriate values.
-
-```yaml
-global:
-  images:
-    tag: "dev"
-    # TODO: We should use the base images here but pythonModulesToInstall would need to be split
-    services: ghcr.io/diracgrid/diracx/services
-    client: ghcr.io/diracgrid/diracx/client
-
-diracx:
-  pythonModulesToInstall:
-    - "git+https://github.com/USERNAME/diracx.git@BRANCH_NAME#egg=diracx_core&subdirectory=diracx-core"
-    - "git+https://github.com/USERNAME/diracx.git@BRANCH_NAME#egg=diracx_db&subdirectory=diracx-db"
-    - "git+https://github.com/USERNAME/diracx.git@BRANCH_NAME#egg=diracx_routers&subdirectory=diracx-routers"yaml
-```
 
 ## OpenTelemetry
 

demo/values.tpl.yaml

@@ -6,6 +6,7 @@ global:
   batchJobTTL: 3600
 
 developer:
+  enabled: true
   urls:
     diracx: https://{{ hostname }}:8000
     minio: http://{{ hostname }}:32000
@@ -22,6 +23,7 @@ diracx:
   hostname: {{ hostname }}
   settings:
     DIRACX_SERVICE_AUTH_TOKEN_ISSUER: "https://{{ hostname }}:8000"
+    DIRACX_CONFIG_BACKEND_URL: "git+file:///cs_store/initialRepo"
     DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS: '["https://{{ hostname }}:8000/api/docs/oauth2-redirect", "https://{{ hostname }}:8000/#authentication-callback"]'
     DIRACX_SANDBOX_STORE_BUCKET_NAME: demo-sandboxes
     DIRACX_SANDBOX_STORE_S3_CLIENT_KWARGS: '{"endpoint_url": "http://{{ hostname }}:32000", "aws_access_key_id": "console", "aws_secret_access_key": "console123"}'
@@ -53,7 +55,7 @@ dex:
     issuer: http://{{ hostname }}:32002
 
     staticClients:
-      - id: d396912e-2f04-439b-8ae7-d8c585a34790
+      - id: {{ dex_client_uuid }}
         public: true
         name: "CLI app"
         redirectURIs:
@@ -62,8 +64,11 @@ dex:
 
     staticPasswords:
       - email: "[email protected]"
+        # bcrypt hash of the string "password"
+        # htpasswd -bnBC 10 "" "password" | tr -d ':\n'
         hash: "$2a$10$2b2cU8CPhOTaGrs1HRQuAueS7JTT5ZHsHSzYiFPm1leZck7Mc8T4W"
         username: "admin"
+        userID: {{ dex_admin_uuid }}
 
 indigoiam:
   config:

diracx/templates/diracx/init-secrets/_init-secrets.sh.tpl

@@ -132,18 +132,6 @@ generate_secret_if_needed diracx-sql-root-connection-urls \
 
 
 
-
-
-
-
-
-
-
-
-
-
-
-
 {{- if .Values.initOs.enabled }}
 # If we deploy opensearch ourselves
 {{- if .Values.opensearch.enabled }}

diracx/templates/diracx/tests/test-connection.yaml

@@ -89,7 +89,7 @@ initKeyStore:
   enabled: true
 
 developer:
-  enabled: true
+  enabled: false
   # -- Make it possible to launch the demo without having an internet connection
   offline: false
   # -- URLs which can be used to access various components of the demo (diracx, minio, dex, etc).
@@ -126,22 +126,48 @@ diracx:
   settings:
     # -- This corresponds to the basic dirac.cfg
     # which must be present on all the servers
-    #TODO: autogenerate all of these
-    DIRACX_CONFIG_BACKEND_URL: "git+file:///cs_store/initialRepo"
+    # -- URL to get the diracx config
+    DIRACX_CONFIG_BACKEND_URL: "git+https://gitlab.invalid/myvo/diracx-config"
+
+    ### AuthSettings https://github.com/DIRACGrid/diracx/blob/main/diracx-routers/src/diracx/routers/utils/users.py
+    # -- path storing the token key
     DIRACX_SERVICE_AUTH_TOKEN_KEYSTORE: "file:///keystore/jwks.json"
     DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS: '["http://anything:8000/docs/oauth2-redirect"]'
 
+    # -- legacy exchange key for DIRAC legacy (see https://github.com/DIRACGrid/diracx/blob/7f766158a674fde0eed011cd2745d359e507f846/diracx-routers/src/diracx/routers/auth/token.py#L264)
+    # -- DIRACX_LEGACY_EXCHANGE_HASHED_API_KEY: <sha256>
+
+    # -- Sandbox settings See https://github.com/DIRACGrid/diracx/blob/7f766158a674fde0eed011cd2745d359e507f846/diracx-routers/src/diracx/routers/jobs/sandboxes.py#L46
+
+    # -- Name of the bucket for the sandbox
+    DIRACX_SANDBOX_STORE_BUCKET_NAME: sandboxes-store
+    DIRACX_SANDBOX_STORE_S3_CLIENT_KWARGS: '{"endpoint_url": "http://minio.invalid:32000", "aws_access_key_id": "my-access-key", "aws_secret_access_key": "my-secret-key-123"}'
+    DIRACX_SANDBOX_STORE_AUTO_CREATE_BUCKET: "true"
+
+    ###
+
+    ### Open Telemetry settings (experimental)
+
+    DIRACX_OTEL_ENABLED: false
+    DIRACX_OTEL_GRPC_ENDPOINT: "diracx-demo-opentelemetry-collector:4317"
+    DIRACX_OTEL_GRPC_INSECURE: "true"
+
   # If mysql is enabled, you are not allowed
   # to set the username passwords
   sqlDbs:
+    # -- default credentials
     default:
     #     rootUser: admin
     #     rootPassword: hunter123
     #     user: dirac
     #     password: password123
     #     host: sqlHost:123
+
     # -- Which DiracX MySQL DBs are used?
     dbs:
+    # All DBs used should be configured here.
+    # If they use the default configuration
+    # they should have a null value
   #    AuthDB:
   #      internalName: DiracXAuthDB
   #    JobDB:
@@ -289,7 +315,7 @@ dex:
   enabled: true
   https.enabled: false
   image:
-    tag: v2.37.0
+    tag: v2.41.1
 
   service:
     type: NodePort
@@ -303,12 +329,10 @@ dex:
 
   config:
     issuer: http://anything:32002
-
     storage:
       type: sqlite3
       config:
         file: /tmp/dex.db
-
     web:
       http: 8000