refactor(config): split into base.ini + env.ini + secrets.ini

Config merge order at startup: base.ini < env.ini < secrets.ini

- deploy/base.ini: common config (baked into image)
- deploy/staging.ini: staging-specific (CI creates ConfigMap)
- deploy/production.ini: prod-specific (CI creates ConfigMap)
- Entrypoint merges all three into /tmp/production.ini
- CI workflow creates ckan-env-config ConfigMap per environment

Author: cooper667

.github/workflows/build-deploy.yml

@@ -108,8 +108,24 @@ jobs:
           echo "${{ secrets.KUBECONFIG_BASE64 }}" | base64 -d > ~/.kube/config
           chmod 600 ~/.kube/config
 
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+
       - name: Deploy to AKS
         run: |
+          # Determine env config file
+          if [[ "${{ steps.params.outputs.namespace }}" == "adr-p" ]]; then
+            ENV_CONFIG="deploy/production.ini"
+          else
+            ENV_CONFIG="deploy/staging.ini"
+          fi
+
+          # Create/update env ConfigMap
+          kubectl create configmap ckan-env-config \
+            --from-file=env.ini=$ENV_CONFIG \
+            -n ${{ steps.params.outputs.namespace }} \
+            --dry-run=client -o yaml | kubectl apply -f -
+
+          # Update image
           kubectl set image deployment/ckan \
             ckan=${{ env.ACR_NAME }}.azurecr.io/${{ env.IMAGE_NAME }}:${{ steps.params.outputs.image_tag }} \
             -n ${{ steps.params.outputs.namespace }}

deploy/Dockerfile.prod

@@ -81,7 +81,7 @@ RUN ln -s /usr/lib/adx /usr/lib/ckan
 
 # Copy entrypoint and config files
 COPY deploy/ckan-entrypoint-prod.sh /ckan-entrypoint.sh
-COPY deploy/production.ini $CKAN_CONFIG/base.ini
+COPY deploy/base.ini $CKAN_CONFIG/base.ini
 COPY ckan/adx_who.ini $CKAN_CONFIG/who.ini
 COPY ckan/ckan_supervisor.conf /etc/supervisor/conf.d/ckan_supervisor.conf
 COPY deploy/uwsgi.ini /usr/lib/adx/uwsgi.ini

deploy/base.ini

@@ -0,0 +1,337 @@
+#
+# CKAN - Pylons configuration
+#
+# These are some of the configuration options available for your CKAN
+# instance. Check the documentation in 'doc/configuration.rst' or at the
+# following URL for a description of what they do and the full list of
+# available options:
+#
+# http://docs.ckan.org/en/latest/maintaining/configuration.html
+#
+# The %(here)s variable will be replaced with the parent directory of this file
+#
+
+[DEFAULT]
+
+# WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PUBLIC ENVIRONMENT*
+# With debug mode enabled, a visitor to your site could execute malicious commands.
+debug = false
+
+[app:main]
+use = egg:ckan
+
+## Development settings
+ckan.devserver.host = localhost
+ckan.devserver.port = 5000
+
+
+## CKAN <2.11 Session settings
+cache_dir = /tmp/%(ckan.site_id)s/
+beaker.session.key = ckan
+beaker.session.type = ext:redis
+beaker.session.url = redis://redis:6379/3
+beaker.session.cookie_expires = 86400
+beaker.session.timeout = 86400
+# This is the secret token that the beaker library uses to hash the cookie sent
+# to the client. `ckan generate config` generates a unique value for this each
+# time it generates a config file.
+# NOTE: Must be set directly - CKAN 2.11 doesn't support separate secrets.ini loading
+beaker.session.secret = ${BEAKER_SESSION_SECRET}
+
+## CKAN >=2.11 Session settings
+## Using default cookie session storage until this issue is resolved:
+## https://github.com/ckan/ckan/issues/8547
+## SESSION_TYPE = redis
+## SESSION_COOKIE_NAME = ckan
+## SESSION_PERMANENT = true
+## PERMANENT_SESSION_LIFETIME = 86400
+# The secret token that is used for session management and other security related tasks as well
+# NOTE: Must be set directly - CKAN 2.11 doesn't support CKAN__SECRET_KEY env var reliably
+SECRET_KEY = ${SECRET_KEY}
+
+# CSRF Protection - enabled by default (WTF_CSRF_ENABLED = True)
+# FileUploader in ckanext-unaids sends X-CSRFToken header
+
+# `ckan generate config` generates a unique value for this each time it generates
+# a config file.
+app_instance_uuid = 853f1fc8-aba1-411e-9071-44e3cdce64dd
+
+# repoze.who config
+who.config_file = %(here)s/who.ini
+who.log_level = warning
+who.log_file = %(cache_dir)s/who_log.ini
+# Session timeout (user logged out after period of inactivity, in seconds).
+# Inactive by default, so the session doesn't expire.
+# who.timeout = 86400
+
+## Database Settings
+
+# PostgreSQL' full-text search parameters
+ckan.datastore.default_fts_lang = english
+ckan.datastore.default_fts_index_method = gist
+
+
+## Site Settings
+
+# ckan.site_url = # configured with env CKAN_SITE_URL
+# ckan.use_pylons_response_cleanup_middleware = true
+
+## Upload Settings
+ckan.upload.user.types = image
+ckan.upload.user.mimetypes = image/png image/bmp image/jpeg image/svg+xml
+
+## Authorization Settings
+
+ckan.auth.anon_create_dataset = false
+ckan.auth.create_unowned_dataset = false
+ckan.auth.create_dataset_if_not_in_organization = false
+ckan.auth.user_create_groups = false
+ckan.auth.user_create_organizations = false
+ckan.auth.user_delete_groups = true
+ckan.auth.user_delete_organizations = true
+ckan.auth.create_user_via_api = false
+ckan.auth.create_user_via_web = true
+ckan.auth.roles_that_cascade_to_sub_groups = admin
+ckan.auth.public_user_details = true
+ckan.auth.public_activity_stream_detail = true
+ckan.auth.allow_dataset_collaborators = true
+ckan.auth.create_default_api_keys = false
+
+## API Token Settings
+api_token.nbytes = 60
+# NOTE: string: prefix is required - CKAN treats bare values as filenames
+# These must be set directly (CKAN 2.11 doesn't load secrets.ini separately)
+api_token.jwt.encode.secret = string:${JWT_SECRET}
+api_token.jwt.decode.secret = string:${JWT_SECRET}
+api_token.jwt.algorithm = HS256
+
+## API Token: expire_api_token plugin
+expire_api_token.default_lifetime = 3600
+
+## Search Settings
+
+ckan.site_id = default
+#solr_url = http://127.0.0.1:8983/solr
+solr_timeout = 60
+
+
+## Redis Settings
+
+# URL to your Redis instance, including the database to be used.
+# ckan.redis.url = redis://localhost:6379/0  # Fjelltopp sets through ENV VAR
+
+
+## CORS Settings
+
+# If cors.origin_allow_all is true, all origins are allowed.
+# If false, the cors.origin_whitelist is used.
+# ckan.cors.origin_allow_all = true
+# cors.origin_whitelist is a space separated list of allowed domains.
+# ckan.cors.origin_whitelist = http://example1.com http://example2.com
+
+## Cache settings that are set by CKAN core
+ckan.cache_enabled = true
+ckan.cache_expires = 604800
+ckan.static_max_age = 604800
+
+## Plugins Settings
+
+ckanext.emailasusername.auto_generate_username_from_fullname = True
+ckanext.emailasusername.require_user_email_input_confirmation = False
+
+# Note: Add ``datastore`` to enable the CKAN DataStore
+#       Add ``datapusher`` to enable DataPusher
+#               Add ``resource_proxy`` to enable resorce proxying and get around the
+#               same origin policy
+ckan.plugins = activity unaids fork scheming_datasets blob_storage emailasusername restricted authz_service stats datastore datapusher validation ytp_request pages harvest versions auth saml2auth
+
+
+# Giftless/LFS server URL - set in env.ini (staging.ini or production.ini)
+# ckanext.blob_storage.storage_service_url = (from env.ini)
+
+ckanext.authz_service.jwt_private_key_file = /etc/ckan/jwt-rs256.key
+ckanext.authz_service.jwt_public_key_file = /etc/ckan/jwt-rs256.key.pub
+
+# Define which views should be created by default
+# (plugins must be loaded in ckan.plugins)
+ckan.views.default_views = geojson_view unaids_recline_view pdf_view image_view text_view
+
+# Customize which text formats the text_view plugin will show
+#ckan.preview.json_formats = json
+#ckan.preview.xml_formats = xml rdf rdf+xml owl+xml atom rss
+#ckan.preview.text_formats = txt plain text/plain
+
+# Customize which image formats the image_view plugin will show
+#ckan.preview.image_formats = png jpeg jpg gif
+
+## Front-End Settings
+
+ckan.site_title = The AIDS Data Repository
+ckan.site_logo = /adr_simple.png
+ckan.site_description = Data Catalogue
+ckan.favicon = /images/favicon.ico
+ckan.gravatar_default = identicon
+ckan.preview.direct = png jpg gif
+ckan.preview.loadable = html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json
+ckan.display_timezone = server
+
+# package_hide_extras = for_search_index_only
+#package_edit_return_url = http://another.frontend/dataset/<NAME>
+#package_new_return_url = http://another.frontend/dataset/<NAME>
+#licenses_group_url = http://licenses.opendefinition.org/licenses/groups/ckan.json
+# ckan.template_footer_end =
+
+
+## Internationalisation Settings
+ckan.locale_default = en
+ckan.locale_order = en fr pt_PT
+ckan.locales_offered = en fr pt_PT
+ckan.locales_filtered_out = en_GB
+
+## Feeds Settings
+
+ckan.feeds.authority_name =
+ckan.feeds.date =
+ckan.feeds.author_name =
+ckan.feeds.author_link =
+
+## Storage Settings
+
+#ckan.storage_path = /var/lib/ckan
+ckan.max_resource_size = 100
+#ckan.max_image_size = 2
+
+## Webassets Settings
+ckan.webassets.use_x_sendfile = true
+ckan.webassets.path = /var/lib/ckan/webassets
+
+
+## Datapusher settings
+
+# Make sure you have set up the DataStore
+
+ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet geojson
+ckan.datapusher.url = http://datapusher:8800
+# Generate a real token: ckan -c /etc/ckan/production.ini user token add admin datapusher
+ckan.datapusher.api_token = ${CKAN_DATAPUSHER_API_TOKEN}
+ckan.datapusher.callback_url_base = http://ckan:5000/
+ckan.datapusher.assume_task_stale_after = 3600
+
+# Resource Proxy settings
+# Preview size limit, default: 1MB
+#ckan.resource_proxy.max_file_size = 1048576
+# Size of chunks to read/write.
+#ckan.resource_proxy.chunk_size = 4096
+
+## Activity Streams Settings
+
+
+
+## Background Job Settings
+
+# Additional CKAN settings for extensions, configurable via group_vars
+# CKAN
+ckan.resource_formats = /usr/lib/ckan/submodules/ckanext-unaids/ckanext/unaids/resource_formats.json
+# ckan.route_after_login = validate_user_affiliation.check_user_affiliation
+ckan.jobs.timeout = 500
+ckan.datasets_per_page = 70
+ckan.group_and_organization_list_all_fields_max = 1000
+ckan.group_and_organization_list_max = 1000
+
+# Scheming
+scheming.dataset_schemas_directory = /usr/lib/ckan/submodules/unaids_data_specifications/package_schemas
+scheming.presets = ckanext.unaids:presets.json
+                  ckanext.scheming:presets.json
+                  ckanext.validation:presets.json
+
+# Blob storage
+ckanext.blob_storage.use_scheming_file_uploader = True
+
+# Validation
+ckanext.unaids.schema_directory = /usr/lib/ckan/submodules/unaids_data_specifications/table_schemas
+ckanext.validation.run_on_create_async = True
+ckanext.validation.run_on_update_async = True
+ckanext.validation.run_on_create_sync = False
+ckanext.validation.run_on_update_sync = False
+ckanext.validation.allow_invalid_data = True
+ckanext.validation.formats = csv xlsx xls geojson
+ckanext.validation.default_validation_options = {"limit_rows": 50000, "limit_errors": 100, "skip_errors": ["extra-header"], "schema_sync": true}
+
+# Email as username
+emailasusername.search_by_username_and_email = True
+
+# Pages/CMS
+ckanext.pages.editor = ckeditor
+ckanext.pages.allow_html = True
+ckanext.pages.about_menu = False
+ckanext.pages.group_menu = False
+ckanext.pages.blog = False
+
+# Harvest
+ckan.harvest.mq.hostname = redis
+ckan.harvest.mq.type = redis
+ckan.harvest.mq.port = 6379
+ckan.harvest.mq.redis_db = 2
+
+# Spatial
+ckanext.spatial.common_map.type = custom
+ckanext.spatial.common_map.custom.url = https://tile.openstreetmap.org/{z}/{x}/{y}.png
+ckanext.spatial.common_map.attribution = &copy; <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a> contributors
+
+# SAML2.0 - Auth0 IDP
+ckanext.saml2auth.idp_metadata.location = remote
+ckanext.saml2auth.idp_metadata.remote_url = https://dev-udfgla0l.eu.auth0.com/samlp/metadata/txdzFtnQXamucc3PhJGIoqxoPnRxZY6K
+ckanext.saml2auth.idp_metadata.remote_cert = /tmp/saml_idp.crt
+ckanext.saml2auth.user_fullname = http://schemas.xmlsoap.org/ws/2005/05/identity/claims/fullname
+ckanext.saml2auth.user_email = http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress
+ckanext.saml2auth.want_response_signed = False
+ckanext.saml2auth.want_assertions_signed = True
+ckanext.saml2auth.logout_requests_signed = False
+
+# OAuth2
+ckanext.unaids.oauth2_required_scope = access:adr
+
+# Recaptcha - DISABLED for staging
+# ckan.recaptcha.publickey = <SET_IN_SECRETS>
+# ckan.recaptcha.privatekey = <SET_IN_SECRETS>
+
+## Logging configuration
+[loggers]
+keys = root, ckan, ckanext, werkzeug
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console
+
+[logger_werkzeug]
+level = WARNING
+handlers = console
+qualname = werkzeug
+propagate = 0
+
+[logger_ckan]
+level = INFO
+handlers = console
+qualname = ckan
+propagate = 0
+
+[logger_ckanext]
+level = DEBUG
+handlers = console
+qualname = ckanext
+propagate = 0
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s