Migration updates for CKAN 2.11 and Python 3.10

A-Souhei · A-Souhei · commit d15bfc3f4c16 · 2026-01-13T18:29:25.000+03:00
diff --git a/.gitmodules b/.gitmodules
@@ -46,3 +46,13 @@
 [submodule "ckan-extensions/ckanext-validation"]
     path = submodules/ckanext-validation
 	url = git@github.com:fjelltopp/ckanext-validation.git
+[submodule "submodules/ckanext-authz-service"]
+	path = submodules/ckanext-authz-service
+	url = https://github.com/datopian/ckanext-authz-service.git
+[submodule "submodules/ckanext-pages"]
+	path = submodules/ckanext-pages
+	url = https://github.com/ckan/ckanext-pages.git
+[submodule "submodules/ckanext-saml2auth"]
+	path = submodules/ckanext-saml2auth
+	url = https://github.com/fjelltopp/ckanext-saml2auth.git
+	branch = toavina/ckan-2.11-migration
diff --git a/Pipfile b/Pipfile
@@ -45,7 +45,6 @@ webassets = "==2.0"
 webencodings = "==0.5.1"
 werkzeug = {version = "==3.0.6", extras = ["watchdog"]}
 "zope.interface" = "==6.4.post2"
-
 # Submodules
 ckanext-auth = {editable = true, path = "./submodules/ckanext-auth"}
 ckanext-authz-service = {editable = true, path = "./submodules/ckanext-authz-service"}
@@ -60,15 +59,14 @@ ckanext-validation = {editable = true, path = "./submodules/ckanext-validation"}
 ckanext-blob-storage = {editable = true, path = "./submodules/ckanext-blob-storage"}
 ckanext-ytp-request = {editable = true, path = "./submodules/ckanext-ytp-request"}
 ckanext-fork = {editable = true, path = "./submodules/ckanext-fork"}
+ckanext-saml2auth = {editable = true, path = "./submodules/ckanext-saml2auth"}
 ckan = {editable = true, path = "./submodules/ckan"}
 # end of submodules
-
 # Additional CKAN extensions (local submodule)
 ckanext-pages = {editable = true, path = "./submodules/ckanext-pages"}
-
 # Additional ADX dependencies
 ckanapi = "==4.3"
-ckanext-saml2auth = "==1.3.0"
+pysaml2 = ">=6.5.1,<7.4"
 "repoze.who" = "==2.3"
 importlib-resources = "*"
 cffi = "==1.14.5"
@@ -90,7 +88,7 @@ jsonschema = "==3.2.0"
 charset-normalizer = "==3.3.2"
 python-dotenv = "==1.0.0"
 python-jose = "==3.3.0"
-setuptools = "==58.1.0"
+setuptools = ">=65.0.0"
 raven = "==6.10.0"
 tableschema = "==1.20.2"
 frictionless = ">=5.0.0,<6.0.0"
diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/README.md b/README.md
@@ -141,6 +141,16 @@ git submodule foreach "git checkout development|| :"
 git submodule foreach "git pull || :"
 ```
 
+## Cleaning build artifacts
+
+If you encounter issues with `pipenv lock` failing due to permission errors on `.egg-info` directories (especially after running containers as root), you can clean these build artifacts:
+
+```shell
+adx clean
+```
+
+This command will remove all `.egg-info` directories from the submodules, using `sudo` if necessary when files are owned by root.
+
 ## Using a fake SMTP server
 
 We're using [rnwood/smtp4dev](https://github.com/rnwood/smtp4dev) to "fake" an SMTP service, it's deployed as part of docker compose - smtp container. It catches all the emails sent to it, accepts any credentials. Emails can be viewed via a web console available at port 5555, so if your local environment uses "adr.local" as a host name you can access at [http://adr.local:5555/](http://adr.local:5555/).
diff --git a/adx b/adx
@@ -140,6 +140,13 @@ actions['test'].add_argument(
     const=""
 )
 
+actions['clean'] = subparsers.add_parser(
+    'clean',
+    description='Clean up build artifacts like egg-info directories from submodules.',
+    help='Clean up build artifacts like egg-info directories from submodules.'
+)
+actions['clean'].set_defaults(func=util.clean_build_artifacts)
+
 parser.add_argument(
     "-log",
     "--log",
diff --git a/ckan/Dockerfile b/ckan/Dockerfile
@@ -22,8 +22,6 @@ RUN update-locale LANG=${LC_ALL}
 ## Install required system packages
 RUN apt-get -q -y update \
     && apt-get -q -y install \
-        python-wheel-common \
-        python3-distutils \
         libpq-dev \
         libxml2-dev \
         libxslt-dev \
@@ -39,11 +37,10 @@ RUN apt-get -q -y update \
         xmlsec1 \
         jq \
         supervisor \
-        jq \
     && apt-get -q clean \
     && rm -rf /var/lib/apt/lists/*
 RUN pip3 install pipenv
-RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - && apt-get install nodejs npm -y && npm version
+RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - && apt-get install nodejs -y && npm version
 RUN npm install --global yarn
 RUN mkdir -p /var/lib/ckan/resources && chmod 777 -R /var/lib/ckan
 
diff --git a/ckan/adx_config.ini b/ckan/adx_config.ini
@@ -124,10 +124,10 @@ ckan.redis.url = redis://redis:6379/1
 #   Add ``resource_proxy`` to enable resorce proxying and get around the
 #   same origin policy
 # Note: Plugins to the left take precendence over plugins to the right!!! Opposite to what you might expect.
-ckan.plugins = unaids fork scheming_datasets blob_storage saml2auth emailasusername restricted authz_service stats
-  text_view image_view unaids_recline_view recline_graph_view recline_map_view recline_grid_view
-  resource_proxy geo_view pdf_view datastore datapusher geojson_view
-  validation ytp_request pages dhis2harvester_plugin dhis2_pivot_tables_harvester harvest sentry versions
+ckan.plugins = activity unaids fork scheming_datasets blob_storage saml2auth emailasusername restricted authz_service stats
+  text_view image_view unaids_recline_view
+  resource_proxy datastore datapusher
+  validation ytp_request pages dhis2harvester_plugin dhis2_pivot_tables_harvester harvest versions
   auth
 
 ckanext.blob_storage.storage_service_url=http://adr.local/giftless
@@ -202,7 +202,7 @@ ckan.harvest.mq.port=6379
 ckan.harvest.mq.redis_db=2
 # Define which views should be created by default
 # (plugins must be loaded in ckan.plugins)
-ckan.views.default_views = geojson_view unaids_recline_view pdf_view image_view text_view
+ckan.views.default_views = unaids_recline_view image_view text_view
 
 # Customize which text formats the text_view plugin will show
 ckan.preview.json_formats = json
@@ -273,6 +273,7 @@ ckan.max_resource_size = 256
 
 ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet geojson
 ckan.datapusher.url = http://datapusher:8800/
+ckan.datapusher.api_token = dev-datapusher-token-change-in-production
 ckan.datapusher.assume_task_stale_after = 3600
 
 # Resource Proxy settings
diff --git a/ckan/ckan-entrypoint-adx.sh b/ckan/ckan-entrypoint-adx.sh
@@ -75,4 +75,14 @@ chmod -R 777 /usr/lib/adx/submodules/ckanext-unaids/ckanext/unaids/assets/build
 set_environment
 echo "CKAN bootstrapping finished, environment ready"
 
+# Initialize CKAN database and run plugin migrations
+echo "Initializing CKAN database..."
+ckan --config="$CONFIG" db init || echo "CKAN database already initialized"
+
+echo "Running database migrations for plugins..."
+ckan --config="$CONFIG" db upgrade -p pages || echo "Warning: ckanext-pages migration failed or already applied"
+ckan --config="$CONFIG" versions initdb || echo "Warning: ckanext-versions initdb failed or already applied"
+ckan --config="$CONFIG" validation init-db || echo "Warning: ckanext-validation init-db failed or already applied"
+ckan --config="$CONFIG" unaids initdb || echo "Warning: ckanext-unaids initdb failed or already applied"
+
 exec "$@"
diff --git a/db/Dockerfile b/db/Dockerfile
@@ -1,4 +1,4 @@
-FROM postgres:12-bullseye
+FROM postgres:16
 MAINTAINER Fjelltopp
 
 RUN apt-get update \
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -11,10 +11,17 @@ services:
   ckan:
     container_name: ckan
     image: ghcr.io/fjelltopp/adx_develop/ckan_base:${CKAN_IMAGE_TAG}
+    build:
+      context: ./ckan
+      dockerfile: Dockerfile
     deploy:
       resources:
         limits:
           memory: 6G
+    depends_on:
+      - db
+      - solr
+      - redis
     links:
       - db
       - solr
@@ -56,6 +63,9 @@ services:
   supervisor:
     container_name: supervisor
     image: ghcr.io/fjelltopp/adx_develop/ckan_base:${CKAN_IMAGE_TAG}
+    build:
+      context: ./ckan
+      dockerfile: Dockerfile
     depends_on:
       - ckan
     environment:
diff --git a/solr/Dockerfile b/solr/Dockerfile
@@ -1,4 +1,4 @@
-FROM ckan/ckan-solr:2.9-solr8
+FROM ckan/ckan-solr:2.11-solr9
 MAINTAINER CKAN
 
 # Enviroment
diff --git a/submodules/ckanext-authz-service b/submodules/ckanext-authz-service
@@ -0,0 +1 @@
+Subproject commit d179b19469faff7e461191db53e31c60d19003bc
diff --git a/submodules/ckanext-pages b/submodules/ckanext-pages
@@ -0,0 +1 @@
+Subproject commit 6c8d5939b01964c30a0394f10fef6e8f5a210ada
diff --git a/submodules/ckanext-saml2auth b/submodules/ckanext-saml2auth
@@ -0,0 +1 @@
+Subproject commit 791a6fc7a2dfd17fd3676f4923603ef5795cded2
diff --git a/util/__init__.py b/util/__init__.py
@@ -182,3 +182,66 @@ def deploy_master(args, extra):
             ADX_PATH
         )
     ])
+
+def clean_build_artifacts(args, extra):
+    """
+    Clean up build artifacts like egg-info directories from submodules.
+    This is useful when directories were created with different permissions
+    (e.g., by root in Docker containers) and are blocking pipenv lock.
+    
+    Args:
+        args (NameSpace): The known args NameSpace object returned by argsparse
+        extra (list): The extra args not parsed by argsparse
+    """
+    import glob
+    import shutil
+    import stat
+    
+    submodules_path = os.path.join(COMPOSE_PATH, 'submodules')
+    
+    if not os.path.exists(submodules_path):
+        print(f"Submodules directory not found: {submodules_path}")
+        return
+    
+    # Find all .egg-info directories
+    egg_info_dirs = glob.glob(os.path.join(submodules_path, '*', '*.egg-info'))
+    
+    if not egg_info_dirs:
+        print("No egg-info directories found to clean.")
+        return
+    
+    print(f"Found {len(egg_info_dirs)} egg-info directories to clean:")
+    for egg_dir in egg_info_dirs:
+        print(f"  - {os.path.relpath(egg_dir, COMPOSE_PATH)}")
+    
+    # Check if any are owned by root or not writable
+    need_sudo = False
+    for egg_dir in egg_info_dirs:
+        try:
+            # Try to check if we can write to it
+            if not os.access(egg_dir, os.W_OK):
+                need_sudo = True
+                break
+        except Exception:
+            need_sudo = True
+            break
+    
+    if need_sudo:
+        print("\nSome directories require elevated permissions.")
+        print("Attempting to remove with sudo...")
+        retcode = call_command([
+            f'sudo rm -rf {os.path.join(submodules_path, "*", "*.egg-info")}'
+        ])
+        if retcode == 0:
+            print("✓ Successfully cleaned all egg-info directories.")
+        else:
+            print("✗ Failed to clean some directories.")
+    else:
+        # Remove without sudo
+        for egg_dir in egg_info_dirs:
+            try:
+                shutil.rmtree(egg_dir)
+                print(f"✓ Removed {os.path.relpath(egg_dir, COMPOSE_PATH)}")
+            except Exception as e:
+                print(f"✗ Failed to remove {os.path.relpath(egg_dir, COMPOSE_PATH)}: {e}")
+        print("✓ Cleaning complete.")

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-FROM postgres:12-bullseye`
	`1`	`+FROM postgres:16`
`2`	`2`	`MAINTAINER Fjelltopp`
`3`	`3`
`4`	`4`	`RUN apt-get update \`