Config declaration

.github/workflows/ci.yml

@@ -8,19 +8,16 @@ on:
   pull_request:
     branches: main
 
-env:
-  CKANVERSION: 2.9
-
 jobs:
   code_quality:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
+          python-version: '3.10'
 
       - name: Install flake8
         run: |
@@ -29,107 +26,83 @@ jobs:
 
       - name: Lint with flake8
         run: |
-          flake8 . --count --max-complexity=12 --max-line-length=127 --statistics --exclude ckan,ckanext-s3filestore
+          flake8 . --count --max-complexity=13 --max-line-length=127 --statistics
 
   test:
-    runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ '3.8' ]
-    name: Python ${{ matrix.python-version }} extension test
+        python-version: ["3.9", "3.10"]
+        ckan-version: ["2.10", "2.11"]
+      fail-fast: false
 
+    name: CKAN ${{ matrix.ckan-version }} Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    container:
+      image: ckan/ckan-dev:${{ matrix.ckan-version }}
+      options: --user root
     services:
-      postgresql:
-        image: postgres
+      solr:
+        image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9
+      postgres:
+        image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
         env:
+          POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
         options: >-
           --health-cmd pg_isready
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-        ports:
-          - 5432:5432
-
       redis:
-        image: redis
+        image: redis:3
+      moto:
+        image: motoserver/moto:latest
+        ports:
+          - 9000:5000
         options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
+          --health-cmd "curl -f http://localhost:5000 || exit 1"
+          --health-interval 5s
+          --health-timeout 2s
           --health-retries 5
-        ports:
-          - 6379:6379
 
-      ckan-solr:
-        # Workflow level env variables are not addressable on job level, only on steps level
-        # image: ghcr.io/keitaroinc/ckan-solr-dev:{{ env.CKANVERSION }}
-        image: ghcr.io/keitaroinc/ckan-solr-dev:2.9
-        ports:
-          - 8983:8983
+    env:
+      CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
+      CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
+      CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test
+      CKAN_SOLR_URL: http://solr:8983/solr/ckan
+      CKAN_REDIS_URL: redis://redis:6379/1
+      CKANEXT_S3FILESTORE__HOST_NAME: http://moto:5000
+      AWS_ACCESS_KEY_ID: test-access-key
+      AWS_SECRET_ACCESS_KEY: test-secret-key
+      AWS_DEFAULT_REGION: us-east-1
 
     steps:
-      - uses: actions/checkout@v2
-
-      - name: Setup Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          architecture: x64
+      - uses: actions/checkout@v4
 
-      - name: Install pytest
+      - name: Install requirements
         run: |
-          python -m pip install --upgrade pip
-          pip install pytest
+          pip install -e .
+          pip install boto3
+          # Replace default path to CKAN core config file with the one on the container
+          sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
 
-      - name: Setup CKAN
-        env:
-          PGPASSWORD: postgres
+      - name: Create S3 bucket in Moto
         run: |
-          bash bin/setup-ckan.bash
+          python - <<'EOF'
+          import boto3, os
+          s3 = boto3.client(
+              "s3",
+              endpoint_url=os.environ["CKANEXT_S3FILESTORE__HOST_NAME"],
+              aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
+              aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
+              region_name=os.environ["AWS_DEFAULT_REGION"],
+          )
+          s3.create_bucket(Bucket="test-bucket")
+          print("✅ Created test-bucket in Moto")
+          EOF
+
+      - name: Run all tests
+        run: pytest --ckan-ini=test.ini --disable-warnings ckanext/s3filestore/tests
 
-      - name: Test with pytest
-        run: |
-          pytest --ckan-ini=subdir/test.ini --cov=ckanext.s3filestore --disable-warnings ckanext/s3filestore/tests
 
-      - name: Coveralls
-        uses: AndreMiras/coveralls-python-action@develop
-        with:
-          parallel: true
-          flag-name: Python ${{ matrix.python-version }} Unit Test
-
-  publish:
-    needs: test
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Setup Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.8'
-
-      - name: Install setup requirements
-        run: |
-          python -m pip install --upgrade setuptools wheel twine
-
-      - name: Build and package
-        run: |
-          python setup.py sdist bdist_wheel
-          twine check dist/*
-
-      - name: Publish package
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          user: __token__
-          password: ${{ secrets.PYPI_API_TOKEN }}
-
-  coveralls_finish:
-    needs: test
-    runs-on: ubuntu-latest
-    steps:
-    - name: Coveralls Finished
-      uses: AndreMiras/coveralls-python-action@develop
-      with:
-        parallel-finished: true

README.rst

@@ -9,11 +9,12 @@
 .. image:: https://coveralls.io/repos/github/keitaroinc/ckanext-s3filestore/badge.svg?branch=main
      :target: https://coveralls.io/github/keitaroinc/ckanext-s3filestore?branch=main
 
-.. image:: https://img.shields.io/badge/python-3.8-blue.svg
-    :target: https://www.python.org/downloads/release/python-384/
+.. image:: https://img.shields.io/badge/python-3.8%20|%203.9%20|%203.10%20-blue
+    :target: https://www.python.org/downloads/
+
+.. image:: https://img.shields.io/badge/ckan-2.9%20|%202.10%20|%202.11-yellow
+    :target: https://www.ckan.org/
 
-.. image:: https://img.shields.io/pypi/v/ckanext-s3filestore
-    :target: https://pypi.org/project/ckanext-s3filestore
 
 
 

ckanext/s3filestore/config_declaration.yaml

@@ -0,0 +1,47 @@
+version: 1
+groups:
+  - annotation: ckanext-s3filestore settings
+    options:
+      - key: ckanext.s3filestore.aws_bucket_name
+        example: s3-filestore-test
+        default: 
+        description: |
+            Defines the name of the S3 bucket that serves as the CKAN storage
+            backend, used for persisting and managing all uploaded files
+        required: true
+      - key: ckanext.s3filestore.aws_access_key_id
+        example: Your-Access-Key-ID
+        default: 
+        description: |
+            Specifies the AWS Access Key ID used for
+            authenticating requests to the S3 bucket
+        required: true
+      - key: ckanext.s3filestore.aws_secret_access_key
+        example: Your-Secret-Access-Key
+        default: 
+        description: |
+            Specifies the AWS Secret Access Key associated with the IAM
+            credentials used to authenticate access to the S3 bucket.
+        required: true
+      - key: ckanext.s3filestore.region_name
+        example: us-east-2
+        default: 
+        description: |
+            Specifies the AWS region in which the target 
+            S3 bucket is provisioned
+        required: true
+      - key: ckanext.s3filestore.signature_version
+        example: s3v4
+        default: 
+        description: |
+            Defines the AWS Signature Version to be used for request authentication
+            when accessing the S3 bucket
+        required: true
+      - key: ckanext.s3filestore.aws_use_ami_role
+        example: true
+        type: bool
+        default: 
+        description: |
+            Indicates whether the S3 client should use the IAM role
+            attached to the Amazon EC2 instance or authentication
+            instead of explicit AWS credentials

ckanext/s3filestore/plugin.py

@@ -72,3 +72,7 @@ def get_blueprint(self):
 
     def get_commands(self):
         return [upload_resources, upload_assets]
+
+
+if toolkit.check_ckan_version(min_version="2.10"):
+    S3FileStorePlugin = toolkit.blanket.config_declarations(S3FileStorePlugin)

ckanext/s3filestore/tests/test_blueprints.py

@@ -7,6 +7,7 @@
 from ckantoolkit import config
 import ckan.tests.factories as factories
 from ckan.lib.helpers import url_for
+import re
 
 
 @pytest.mark.usefixtures(u'clean_db', u'clean_index')
@@ -19,12 +20,12 @@ def setup_class(cls):
     def test_resource_download_url(self, resource_with_upload):
         u'''The resource url is expected for uploaded resource file.'''
 
-        expected_url = u'http://test.ckan.net/dataset/{0}/' \
+        expected_url = u'/dataset/{0}/' \
                        u'resource/{1}/download/test.csv'.\
             format(resource_with_upload[u'package_id'],
                    resource_with_upload[u'id'])
 
-        assert resource_with_upload['url'] == expected_url
+        assert re.sub(r'^https?://[^/]+', '', resource_with_upload['url']) == expected_url
 
     def test_resource_download(self, app, resource_with_upload):
         u'''When trying to download resource

ckanext/s3filestore/tests/test_fix_for_webpageview_plugin.py

@@ -49,9 +49,10 @@ def test_view_shown_for_url_type_upload(app, create_with_upload):
 
     response = app.get(url)
 
-    assert (u'/dataset/{0}/resource/{1}/download?preview=True'
-            .format(dataset[u'id'], resource[u'id'])
-            in response)
+    formatted_url = (
+        '/dataset/{0}/resource/{1}/download'.format(dataset['id'], resource['id'])
+    )
+    assert formatted_url in response.body, f"{formatted_url} not found in response"
 
     resource_view_src = app.get(resource_view_src_url,
                                 follow_redirects=False)

conftest.py

@@ -2,6 +2,6 @@
 
 pytest_plugins = [
     u'ckanext.s3filestore.tests.fixtures',
-    u'ckan.tests.pytest_ckan.ckan_setup',
-    u'ckan.tests.pytest_ckan.fixtures',
+    # 'ckan.tests.pytest_ckan.ckan_setup',
+    # u'ckan.tests.pytest_ckan.fixtures',
 ]

test.ini

@@ -23,7 +23,7 @@ ckan.plugins = s3filestore
 ckanext.s3filestore.aws_access_key_id = test-access-key
 ckanext.s3filestore.aws_secret_access_key = test-secret-key
 ckanext.s3filestore.aws_bucket_name = test-bucket
-ckanext.s3filestore.host_name = http://127.0.0.1:9000
+ckanext.s3filestore.host_name = http://moto:5000
 ckanext.s3filestore.region_name = us-east-1
 ckanext.s3filestore.signature_version = s3v4
 ckanext.s3filestore.acl = private