diff --git a/.circleci/config.yml b/.circleci/config.yml index 42f9d51a7a2..0d17e1ef547 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,10 +2,6 @@ version: 2 defaults: init_environemnt: &init_environment run: | - # SOLR config - cp ~/project/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml - service jetty9 restart || true # erroring out but does seem to work - # Database Creation psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_POSTGRES_USER} WITH PASSWORD '${CKAN_POSTGRES_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;" createdb --encoding=utf-8 --host=ckan-postgres --username=ckan --owner=${CKAN_POSTGRES_USER} ${CKAN_POSTGRES_DB} @@ -15,53 +11,20 @@ defaults: # Database Initialization ckan -c test-core-circle-ci.ini datastore set-permissions | psql --host=ckan-postgres --username=ckan + psql --host=ckan-postgres --username=ckan --dbname=${CKAN_DATASTORE_POSTGRES_DB} --command="CREATE extension tablefunc;" ckan -c test-core-circle-ci.ini db init + gunzip .test_durations.gz install_deps: &install_deps run: | # OS Dependencies apt update - case $CIRCLE_NODE_INDEX in - $NODE_TESTS_CONTAINER) - curl -sL https://deb.nodesource.com/setup_10.x | bash - - apt install -y nodejs - apt install -y libgtk2.0-0 libgtk-3-0 libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb - npm install - ;; - esac - apt install -y postgresql-client solr-jetty openjdk-8-jdk + apt install -y postgresql-client run_tests: &run_tests # Tests Backend, split across containers by segments run: | - mkdir -p ~/junit - case $CIRCLE_NODE_INDEX in - 0) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 1 - ;; - 1) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 2 - ;; - 2) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 3 - ;; - 3) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 4 - ;; - esac + mkdir -p ~/junit/result + python -m pytest ${PYTEST_COMMON_OPTIONS} --splits 4 --group $((CIRCLE_NODE_INDEX+1)) --splitting-algorithm least_duration - # Tests Frontend, only in one container - start_test_server: &start_test_server - run: - command: | - case $CIRCLE_NODE_INDEX in - $NODE_TESTS_CONTAINER) ckan -c test-core-circle-ci.ini run - ;; - esac - background: true - run_front_tests: &run_front_tests - run: - command: | - case $CIRCLE_NODE_INDEX in - $NODE_TESTS_CONTAINER) - sleep 5 - $(npm bin)/cypress run - ;; - esac ckan_env: &ckan_env environment: CKAN_DATASTORE_POSTGRES_DB: datastore_test @@ -73,8 +36,7 @@ defaults: CKAN_POSTGRES_USER: ckan_default CKAN_POSTGRES_PWD: pass PGPASSWORD: ckan - NODE_TESTS_CONTAINER: 3 - PYTEST_COMMON_OPTIONS: -v --ckan-ini=test-core-circle-ci.ini --cov=ckan --cov=ckanext --junitxml=/root/junit/junit.xml --test-group-count 4 --test-group-random-seed 1 + PYTEST_COMMON_OPTIONS: -v --ckan-ini=test-core-circle-ci.ini --cov=ckan --cov=ckanext --junitxml=~/junit/result/junit.xml pg_image: &pg_image image: postgres:10 environment: @@ -85,13 +47,19 @@ defaults: redis_image: &redis_image image: redis:3 name: ckan-redis + + solr_image: &solr_image + image: ckan/ckan-solr:master + name: ckan-solr + jobs: test-python-3: docker: - - image: python:3-stretch + - image: python:3.7-bullseye <<: *ckan_env - <<: *pg_image - <<: *redis_image + - <<: *solr_image parallelism: 4 @@ -101,7 +69,6 @@ jobs: - <<: *install_deps - run: | # Python Dependencies - pip install -r requirement-setuptools.txt pip install -r requirements.txt pip install -r dev-requirements.txt python setup.py develop @@ -112,10 +79,6 @@ jobs: - <<: *run_tests - store_test_results: path: ~/junit - - <<: *start_test_server - - <<: *run_front_tests - - store_artifacts: - path: ~/project/cypress/screenshots - run: coveralls workflows: version: 2 diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 00000000000..a0614363461 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,23 @@ +# [Choice] Python version: 3, 3.8, 3.7, 3.6 +ARG VARIANT=3 +FROM mcr.microsoft.com/vscode/devcontainers/python:${VARIANT} + +ENV PYTHONUNBUFFERED 1 + +# Update args in docker-compose.yaml to set the UID/GID of the "vscode" user. +ARG USER_UID=1000 +ARG USER_GID=$USER_UID +RUN if [ "$USER_GID" != "1000" ] || [ "$USER_UID" != "1000" ]; then groupmod --gid $USER_GID vscode && usermod --uid $USER_UID --gid $USER_GID vscode; fi + +# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10 +ARG NODE_VERSION="none" +RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi + +# [Optional] If your requirements rarely change, uncomment this section to add them to the image. +# COPY requirements.txt /tmp/pip-tmp/ +# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \ +# && rm -rf /tmp/pip-tmp + + RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends \ + postgresql-client diff --git a/.devcontainer/README_codespaces.md b/.devcontainer/README_codespaces.md new file mode 100644 index 00000000000..218c568fd89 --- /dev/null +++ b/.devcontainer/README_codespaces.md @@ -0,0 +1,83 @@ +# CKAN in GitHub Codespaces + +Welcome to your cloud development instance of CKAN! + +⌛ If you are not seeing a browser tab with the CKAN homepage to the right or a terminal below that says _"Running CKAN on http://localhost:5000"_, wait a bit, things are getting set up... ⌛ + +Once you see them, you are ready to go! 🚀 + +## What is this? + +This is an online development environment powered by [GitHub Codespaces](https://github.com/features/codespaces). It is a fully functional CKAN instance that you can configure and customize in any way you need. Changes that you make to the source files with the editor as well as changes to the site itself (e.g. creating a dataset or uploading a file) will be persisted until you delete the codespace. + +⚠ī¸ **Note:** GitHub Codespaces have a free tier, currently 120 core-hours (i.e. 60h on the default 2-core VM, 30h on a 4-core VM), but after that you will be charged for usage. Check the [documentation](https://github.com/features/codespaces) for more details. To check your current usage, go to the [Billing page](https://github.com/settings/billing) in your profile. ⚠ī¸ + + +## What can I do with it? + +### Explore + +It's your own CKAN demo site! You can log in using the `ckan_admin` sysadmin user (password `test1234`) which will give you full control of the UI. Try creating an Organization, adding some datasets, uploading data, etc + +* [User Guide](https://docs.ckan.org/en/latest/user-guide.html) +* [Sysadmin Guide](https://docs.ckan.org/en/latest/sysadmin-guide.html) + + +### Customize + +The site has been configured using the default settings that you get in a brand new CKAN instance but you can change any configuration in the `ckan.ini` file. The development server will refresh automatically as soon as you save your changes to reflect the new configuration. + + * [Configuration options reference](https://docs.ckan.org/en/latest/maintaining/configuration.html#ckan-configuration-file) + * [Authorization overview](https://docs.ckan.org/en/latest/maintaining/authorization.html) + +Additionally, you can install as many extensions as you want. Check the extension README for any particular instructions but they all basically follow the same pattern: +1. Open a new terminal in the panel below +2. Clone the extension + ``` + git clone https://github.com/ckan/ckanext-dcat.git + ``` +3. Install the extension + ``` + cd ckanext-dcat + python setup.py develop --user + ``` +3. Install extra requirements (if any) + ``` + pip install -r requirements.txt + ``` +4. Add the plugin(s) to the `ckan.plugins` configuration option in the `ckan.ini` file. + +### Develop + +What you are using right now is an online editor, Visual Studio Code for the Web, which runs in your browser. You can browse the files in the CKAN source code using the tree panel on the left, open one of them and edit it. Once you save your changes, the development server will be restarted automatically. + +You can commit your changes to the branch where you started the codespace in using the "Source Control" icon in the left toolbar. + +* [Getting started with Visual Studio Code](https://code.visualstudio.com/docs/editor/codebasics) +* [CKAN Architecture Overview](https://docs.ckan.org/en/latest/contributing/architecture.html) + +#### Database + +You can run queries against the PostgreSQL database using the "SQLTools" plugin, the database icon in the left toolbar. + +* [SQLTools documentation](https://vscode-sqltools.mteixeira.dev/en/home/#features) + +#### Tests + +To run the automated tests simply add a new terminal to the console below and run the `pytest` command: + +``` +pytest --ckan-ini=test-core.ini ckan ckanext +``` + +Or to run a specific test: + +``` +pytest --ckan-ini=test-core.ini ckan/tests/logic/action/test_create.py::TestMemberCreate::test_group_member_creation +``` + +## I need more! + +* If you are finding the codespace too slow you can change the machine type to add more cores to the VM, but note that this will count towards your free allowed quota. +* If you don't like the editor, you can run the same codespace using your local [Visual Studio Code](https://docs.github.com/en/codespaces/developing-in-codespaces/using-github-codespaces-in-visual-studio-code) or [JetBrains IDE](https://docs.github.com/en/codespaces/developing-in-codespaces/using-github-codespaces-in-your-jetbrains-ide) instances. +* Alternatively you can use a [Docker Compose setup](https://github.com/ckan/ckan-docker) or do a tried and tested [Source Install](https://docs.ckan.org/en/latest/maintaining/installing/install-from-source.html) in your local computer. diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000000..a9102893717 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,70 @@ +// Update the VARIANT arg in docker-compose.yml to pick a Python version: 3, 3.8, 3.7, 3.6 +{ + "name": "CKAN 2.10", + "dockerComposeFile": "docker-compose.yml", + "service": "ckan", + "workspaceFolder": "/workspace", + + // Set *default* container specific settings.json values on container create. + "settings": { + "sqltools.connections": [{ + "name": "CKAN PostgreSQL DB", + "driver": "PostgreSQL", + "previewLimit": 50, + "server": "localhost", + "port": 5432, + "database": "ckan_default", + "username": "ckan_default", + "password": "pass" + }], + "python.pythonPath": "/usr/local/bin/python", + "python.languageServer": "Pylance", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", + "python.formatting.blackPath": "/usr/local/py-utils/bin/black", + "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", + "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", + "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8", + "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", + "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", + "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", + "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint", + "python.testing.pytestPath": "/usr/local/py-utils/bin/pytest" + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "mtxr.sqltools", + "mtxr.sqltools-driver-pg" + ], + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + "forwardPorts": [5000, 5432, 8983, 6379, 8800], + + // Open in browser + "portsAttributes": { + "5000": { + "label": "CKAN", + "onAutoForward": "openPreview" + } + }, + + // Use 'postCreateCommand' to run commands after the container is created. + "onCreateCommand": "pip install --user -r requirements.txt && pip install --user -r dev-requirements.txt", + "postCreateCommand": "./.devcontainer/setup.sh", + "postAttachCommand": "ckan run", + + // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode", + + "customizations": { + "codespaces": { + "openFiles": [ + ".devcontainer/README_codespaces.md" + ] + } + } +} \ No newline at end of file diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 00000000000..5281349118a --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,64 @@ +version: '3' + +services: + ckan: + build: + context: .. + dockerfile: .devcontainer/Dockerfile + args: + # [Choice] Python version: 3, 3.8, 3.7, 3.6 + VARIANT: 3.9 + # [Choice] Node.js version: none, lts/*, 16, 14, 12, 10 + NODE_VERSION: "lts/*" + # On Linux, you may need to update USER_UID and USER_GID below if not your local UID is not 1000. + USER_UID: 1000 + USER_GID: 1000 + + volumes: + - ..:/workspace:cached + + # Overrides default command so things don't shut down after the process ends. + command: sleep infinity + + # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. + + # Uncomment the next line to use a non-root user for all processes. + # user: vscode + + # Use "forwardPorts" in **devcontainer.json** to forward an app port locally. + # (Adding the "ports" property to this file will not forward from a Codespace.) + + db: + image: postgres:14 + restart: unless-stopped + volumes: + - postgres-data:/var/lib/postgresql/data + - ./postgres-docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d + environment: + POSTGRES_USER: ckan_default + POSTGRES_DB: ckan_default + POSTGRES_PASSWORD: pass + network_mode: service:ckan + # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally. + # (Adding the "ports" property to this file will not forward from a Codespace.) + + solr: + container_name: solr + image: ckan/ckan-solr:2.9-solr8 + network_mode: service:ckan + volumes: + - solr-data:/var/solr + + redis: + network_mode: service:ckan + container_name: redis + image: redis:alpine + + datapusher: + container_name: datapusher + network_mode: service:ckan + image: ckan/ckan-base-datapusher:0.0.19 + +volumes: + postgres-data: + solr-data: diff --git a/.devcontainer/postgres-docker-entrypoint-initdb.d/02_create_datastore_db_and_users.sql b/.devcontainer/postgres-docker-entrypoint-initdb.d/02_create_datastore_db_and_users.sql new file mode 100644 index 00000000000..0e9da9e27b6 --- /dev/null +++ b/.devcontainer/postgres-docker-entrypoint-initdb.d/02_create_datastore_db_and_users.sql @@ -0,0 +1,2 @@ +CREATE ROLE datastore_default NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN PASSWORD 'pass'; +CREATE DATABASE datastore_default OWNER ckan_default ENCODING 'utf-8'; diff --git a/.devcontainer/postgres-docker-entrypoint-initdb.d/create_test_db.sql b/.devcontainer/postgres-docker-entrypoint-initdb.d/create_test_db.sql new file mode 100644 index 00000000000..7adf371a841 --- /dev/null +++ b/.devcontainer/postgres-docker-entrypoint-initdb.d/create_test_db.sql @@ -0,0 +1 @@ +CREATE DATABASE ckan_test OWNER ckan_default ENCODING 'utf-8'; diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh new file mode 100755 index 00000000000..890a2f44970 --- /dev/null +++ b/.devcontainer/setup.sh @@ -0,0 +1,29 @@ +# Install CKAN locally +python setup.py develop --user + +# Create ini file +ckan generate config ckan.ini + +# Set up storage +mkdir /workspace/data +ckan config-tool ckan.ini ckan.storage_path=/workspace/data + +# Set up site URL +ckan config-tool ckan.ini ckan.site_url=https://$CODESPACE_NAME-5000.$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN + +# Init DB +ckan db init + +# Create sysadmin user +ckan user add ckan_admin email=admin@example.com password=test1234 +ckan sysadmin add ckan_admin + +# Set up DataStore + DataPusher +ckan config-tool ckan.ini "ckan.datapusher.api_token=$(ckan user token add ckan_admin datapusher | tail -n 1 | tr -d '\t')" +ckan config-tool ckan.ini \ + ckan.datastore.write_url=postgresql://ckan_default:pass@localhost/datastore_default \ + ckan.datastore.read_url=postgresql://datastore_default:pass@localhost/datastore_default \ + ckan.datapusher.url=http://localhost:8800 \ + ckan.datapusher.callback_url_base=http://localhost:5000 \ + "ckan.plugins=activity datastore datapusher datatables_view" +ckan datastore set-permissions | psql $(grep ckan.datastore.write_url ckan.ini | awk -F= '{print $2}') diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000000..28b4b979888 --- /dev/null +++ b/.flake8 @@ -0,0 +1,65 @@ +[flake8] +exclude = + .* + ./contrib + ./bin +per-file-ignores = + test_*.py:E501 +extend-exclude = + ckan/__init__.py,ckan/config/middleware.py + ckan/lib/app_globals.py + ckan/lib/cli.py + ckan/lib/create_test_data.py + ckan/lib/dictization/__init__.py + ckan/lib/dictization/model_dictize.py + ckan/lib/dictization/model_save.py + ckan/lib/email_notifications.py + ckan/lib/hash.py + ckan/lib/jinja_extensions.py + ckan/lib/maintain.py + ckan/lib/navl/validators.py + ckan/lib/plugins.py + ckan/lib/search/__init__.py + ckan/lib/search/index.py + ckan/lib/search/query.py + ckan/logic/action/__init__.py + ckan/logic/action/delete.py + ckan/logic/action/get.py + ckan/logic/action/update.py + ckan/logic/auth/create.py + ckan/logic/auth/delete.py + ckan/logic/auth/get.py + ckan/logic/auth/update.py + ckan/logic/converters.py + ckan/logic/validators.py + ckan/model/__init__.py + ckan/model/dashboard.py + ckan/model/domain_object.py + ckan/model/follower.py + ckan/model/group.py + ckan/model/group_extra.py + ckan/model/license.py + ckan/model/meta.py + ckan/model/misc.py + ckan/model/modification.py + ckan/model/package.py + ckan/model/package_extra.py + ckan/model/package_relationship.py + ckan/model/resource.py + ckan/model/system_info.py + ckan/model/tag.py + ckan/model/task_status.py + ckan/model/term_translation.py + ckan/model/tracking.py + ckan/model/user.py + ckan/model/vocabulary.py + ckan/authz.py + ckanext/datastore/logic/action.py + ckanext/datastore/tests/test_create.py + ckanext/example_idatasetform/plugin.py + ckanext/example_itemplatehelpers/plugin.py + ckanext/multilingual/plugin.py + ckanext/stats/stats.py + ckanext/test_tag_vocab_plugin.py + doc/conf.py + setup.py diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index ff47e273e38..f064f54b87d 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -7,16 +7,17 @@ assignees: '' --- -**CKAN version** +## CKAN version -**Describe the bug** +## Describe the bug A clear and concise description of what the bug is. -**Steps to reproduce** +### Steps to reproduce Steps to reproduce the behavior: -**Expected behavior** +### Expected behavior A clear and concise description of what you expected to happen. -**Additional details** +### Additional details If possible, please provide the full stack trace of the error raised, or add screenshots to help explain your problem. + diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index b02e9f504c6..8b0e5850978 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -9,6 +9,6 @@ contact_links: - name: Security Issues url: mailto:security@ckan.org about: Please report any security related vulnerabilities here. - - name: Ideas Repository - url: https://github.com/ckan/ideas - about: For new feature requests or discussion, please create an issue on the ideas repository. + - name: Ideas and Discussion + url: https://github.com/ckan/ckan/discussions + about: For new feature requests or discussion, please create use the discussion forum. diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml new file mode 100644 index 00000000000..9817290c555 --- /dev/null +++ b/.github/codeql/codeql-config.yml @@ -0,0 +1,4 @@ +name: "CKAN CodeQL config" + +paths-ignore: + - '**/vendor/**' diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 00000000000..f1f3280fa0f --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,71 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +name: "CodeQL" + +on: + push: + branches: [master] + pull_request: + # The branches below must be a subset of the branches above + branches: [master] + schedule: + - cron: '0 3 * * 2' + +permissions: + contents: read + +jobs: + analyze: + permissions: + actions: read # for github/codeql-action/init to get workflow details + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/autobuild to send a status report + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + # Override automatic language detection by changing the below list + # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] + language: ['python', 'javascript'] + # Learn more... + # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + config-file: ./.github/codeql/codeql-config.yml + setup-python-dependencies: false + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml new file mode 100644 index 00000000000..962ad1d5e22 --- /dev/null +++ b/.github/workflows/cypress.yml @@ -0,0 +1,73 @@ +name: Cypress +on: [pull_request] +env: + NODE_VERSION: '16' + PYTHON_VERSION: '3.7' + +permissions: + contents: read + +jobs: + cypress: + runs-on: ubuntu-latest + services: + ckan-postgres: + image: postgres:10 + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + POSTGRES_USER: ckan_default + POSTGRES_PASSWORD: pass + POSTGRES_DB: ckan_test + + ckan-redis: + image: redis + ports: + - 6379:6379 + ckan-solr: + image: ckan/ckan-solr:master + ports: + - 8983:8983 + + env: + CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@localhost/ckan_test + CKAN_SOLR_URL: http://localhost:8983/solr/ckan + CKAN_REDIS_URL: redis://localhost:6379/1 + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: actions/setup-node@v2-beta + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Install python deps + run: pip install -r requirements.txt -r dev-requirements.txt -e. + + - name: Init environment + run: | + ckan -c test-core-cypress.ini db init + + - name: Run Cypress + uses: cypress-io/github-action@v2 + with: + start: ckan -c test-core-cypress.ini run + + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: cypress-screenshots + path: cypress/screenshots + # Test run video was always captured, so this action uses "always()" condition + - uses: actions/upload-artifact@v1 + if: always() + with: + name: cypress-videos + path: cypress/videos diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml new file mode 100644 index 00000000000..b717e139562 --- /dev/null +++ b/.github/workflows/flake8.yml @@ -0,0 +1,22 @@ +name: Lint +on: [pull_request] + +permissions: + contents: read + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Install requirements + run: pip install flake8 pycodestyle + - name: Check syntax + # Stop the build if there are Python syntax errors or undefined names + run: flake8 --count --statistics --show-source + + - name: Warnings + run: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --extend-exclude="" diff --git a/.github/workflows/pyright.yml b/.github/workflows/pyright.yml new file mode 100644 index 00000000000..9bdf68ba038 --- /dev/null +++ b/.github/workflows/pyright.yml @@ -0,0 +1,26 @@ +name: Check types +on: [pull_request] +env: + NODE_VERSION: '16' + PYTHON_VERSION: '3.7' + +permissions: + contents: read + +jobs: + typecheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: actions/setup-node@v2-beta + with: + node-version: ${{ env.NODE_VERSION }} + - name: Install python deps + run: pip install -r requirements.txt -r dev-requirements.txt -e. + - name: Install node deps + run: npm ci + - name: Check types + run: npx pyright diff --git a/.github/workflows/towncrier.yml b/.github/workflows/towncrier.yml new file mode 100644 index 00000000000..97e736926e4 --- /dev/null +++ b/.github/workflows/towncrier.yml @@ -0,0 +1,22 @@ +name: Changelog entries +on: [pull_request] + +permissions: + contents: read + +jobs: + towncrier: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-python@v4 + with: + python-version: '3.7' + cache: 'pip' + - name: Install python deps + run: pip install -r requirements.txt -r dev-requirements.txt -e. + + - name: Check that changelog is updated + run: towncrier check >> $GITHUB_STEP_SUMMARY diff --git a/.gitignore b/.gitignore index 6549151fc47..f798816fbad 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ ckan.egg-info/* sandbox/* dist .mypy_cache +.eggs/* # pylons development.ini* @@ -48,3 +49,6 @@ node_modules/ # docker contrib/docker/.env cypress.env.json + +# cypress +cypress/videos diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000000..4cff79f39ce --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,18 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build documentation in the doc/ directory with Sphinx +sphinx: + configuration: doc/conf.py + +# Additional formats for download +formats: all + +# Optionally set the version of Python and requirements required to build your docs +python: + version: 3.7 + install: + - requirements: requirements-docs.txt diff --git a/.test_durations.gz b/.test_durations.gz new file mode 100644 index 00000000000..a2b0b744f4d Binary files /dev/null and b/.test_durations.gz differ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 61a2e367608..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,42 +0,0 @@ -os: linux -dist: xenial -group: travis_latest -language: python - - -jobs: - include: - - services: - - docker - - cache: - directories: - - ~/docker - - before_install: - - docker build --rm=false -f contrib/docker/postgresql/Dockerfile -t postgresql . - - docker build --rm=false -f contrib/docker/solr/Dockerfile -t solr . - - docker pull redis:latest - - docker build --rm=false -t ckan . - - install: - - docker run -d --name db postgresql - - docker run -d --name solr solr - - docker run -d --name redis redis:latest - - docker run -d --name ckan -p 5000:5000 --link db:db --link redis:redis --link solr:solr ckan - - script: - - docker ps -a - - - python: "3.8" - env: FLAKE8=true - cache: pip - install: pip install flake8 - before_script: - - flake8 --version - # stop the build if there are Python syntax errors or undefined names - - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude ./ckan/include/rjsmin.py,./contrib/cookiecutter/* - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - script: - - true diff --git a/.tx/config b/.tx/config index a1d59e1b676..f98e5d2c784 100644 --- a/.tx/config +++ b/.tx/config @@ -1,13 +1,15 @@ [main] host = https://www.transifex.com -[ckan.2-9] -file_filter = ckan/i18n//LC_MESSAGES/ckan.po -source_file = ckan/i18n/ckan.pot -source_lang = en -type = PO - +[o:okfn:p:ckan:r:2-10] +file_filter = ckan/i18n//LC_MESSAGES/ckan.po +source_file = ckan/i18n/ckan.pot +source_lang = en +type = PO # Namings not quite the same in Transifex and babel: # Transifex vs Babel (& CKAN) # 'sr@Latin' vs 'sr_Latn' trans.sr@latin = ckan/i18n/sr_Latn/LC_MESSAGES/ckan.po +trans.zh_CN = ckan/i18n/zh_Hans_CN/LC_MESSAGES/ckan.po +trans.zh_TW = ckan/i18n/zh_Hant_TW/LC_MESSAGES/ckan.po + diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 06f46f55f1b..5ca93dd0a7a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,786 @@ Changelog .. towncrier release notes start +v.2.10.0 2023-02-15 +=================== + +Overview +-------- +- CKAN 2.10 supports Python 3.7 to 3.10 +- This version requires a requirements upgrade on source installations +- This version requires a database upgrade +- This version does not require a Solr schema upgrade if you are already using the 2.9 schema, + but it is recommended to upgrade to the 2.10 Solr schema. +- Make sure to check the :ref:`migration-notes-2.10` + +Major features +-------------- +- Added **CSRF protection** to the frontend forms to protect against Cross-Site + Request Forgery attacks. This feature is enabled by default in CKAN core, + extensions are excluded from the CSRF protection to give time to update them, + but CSRF protection will be enforced in the future. + To enforce the CSRF protection in extensions you can use + the :ref:`ckan.csrf_protection.ignore_extensions` setting. + See the :ref:`CSRF section ` in the extension best practices + for more information on how to enable it. (`#6920 `_) +- Refactored the **Authentication logic** to use `Flask-login `_ + instead of repoze.who. This has implications on how login sessions are managed (e.g. when and why users + might be logged out) and will affect all plugins that modify the standard authentication process. Please + check the *Migration notes* section below to learn more (`#6560 `_). +- **Configuration declaration**: declare configuration options to ensure + validation and default values. All declared CKAN configuration options + are validated and converted to the expected type during the application + startup. See the *Migration notes* section below to understand the changes + involved and check the :ref:`documentation `. + (`#6467 `_) +- Add **Signals** support to allow subscriptor-based features in extensions. + See :doc:`extensions/signals` (`#5359 `_) +- Add **Blanket implementations**: decorators providing common + implementations of simple interfaces to reduce boilerplate in plugins. See the ``blanket()`` + method in the :doc:`/extensions/plugins-toolkit` (`#5169 + `_) +- Add CLI commands for API Token management (`#5868 + `_) +- The CKAN source code is fully typed now (`#5924 `_) +- Add extensible snippet for resource uploads (`#6226 + `_) +- Migrated to **Bootstrap 5** from v3 for the default CKAN theme. Bootstrap v3 + templates are still available for use by specifying the base template + folder in the configuration (`#6307 + `_):: + + ckan.base_public_folder=public-bs3 + ckan.base_templates_folder=templates-bs3 + +- Removed the **Docker** related files from the main CKAN repository. A brand new official + Docker setup can be found at the `ckan/ckan-docker + `_ repository. (`#7370 + `_) +- Added new command ``ckan shell`` that opens an interactive python shell with + the Flask's application context preloaded (among other useful objects). + (`#6919 `_) +- Added new sub-commands to the ``search-index`` command (`#7044 `_ + and `#7175 `_): + + - ``list-orphans`` lists all public package IDs which exist in the solr + index, but do not exist in the database. + - ``clear-orphans`` clears the search index for all the public orphaned + packages. + - ``list-unindexed`` lists all ununindexed packages +- Add new group command: ``clean``. + Add ``clean users`` command to delete users containing images with formats + not supported in ``ckan.upload.user.mimetypes`` config option. (`#7241 + `_) +- Activities now receive the full dict of the object they refer to in their + ``data`` section. This allows greater flexibility when creating custom + activities from plugins. (`#6557 `_) +- Site maintainers can choose to completely ignore cookie based by using + ``ckan.auth.enable_cookie_auth_in_api``. When set to False, all API requests + must use :ref:`API Tokens `. Note that this is likely to + break some existing JS modules from the frontend that perform API calls, so + it should be used with caution. (`#7088 + `_) +- CKAN now records the last time a user was active on the site. The minimum + interval between records can be controlled with the + :ref:`ckan.user.last_active_interval` config option. (`#6466 + `_) +- :py:class:`~ckan.plugins.toolkit.BaseModel` class for declarative SQLAlchemy + models added to :py:mod:`ckan.plugins.toolkit`. + Models extending ``BaseModel`` class are attached to the SQLAlchemy's + metadata object automatically:: + + from ckan.plugins import toolkit + + class ExtModel(toolkit.BaseModel): + + __tablename__ = "ext_model" + id = Column(String(50), primary_key=True) + ... (`#7351 `_) +- Add dev containers / GitHub Codespaces config (See the `documentation `_ + + +Minor changes +------------- +- Test factories extends SQLAlchemy factory, are available via fixtures and + produce more random entities using faker library. (`#6335 + `_) +- Migrated preprocessor from LESS to SCSS for preliminary work for Bootstrap + upgrade. (`#6175 `_) +- Add ``ckan.plugins.core.plugin_loaded`` to the core helpers as ``plugin_loaded`` + (`#7011 `_) +- Make HTTP response returned on a private dataset if not authorized configurable (`#6641 + `_) +- Allow ``_id`` for ``datastore_upsert`` unique key (`#6793 + `_) +- Add functionality to ``user_show`` to fetch own details when logged in + without passing id (`#5490 `_) +- ``datastore_info`` now returns more detailed info. It returns database-level + metadata in addition + to rowcount (aliases, id, size, index_size, db_size and table_type), and the + data dictionary with + database-level schemata (native_type, index_name, is_index, notnull & + uniquekey). + See the documentation at + :py:func:`~ckanext.datastore.logic.action.datastore_info` (`#5831 + `_) +- ``datastore_info`` now works with aliases, and can be used to dereference + aliases. (`#5832 `_) +- Document new ``ckan.download_proxy`` config value for extensions that download + external URLs (`#xloader-127 + `_) +- Add `organization_followee_count` to the get api (`#2628 + `_) +- Environment variables prefixed with `CKAN_` can be used as variables inside + config file via ``option = %(CKAN_***)s`` (`#6192 + `_) +- CLI command ``less`` is now renamed to ``sass`` as the preprocessor was changed in + #6175. (`#6287 `_) +- Support including file attachments when sending emails (`#6535 + `_) +- Reworked the JavaScript for the view filters to allow for special characters + as well as colons and pipes, which previously caused errors. Added a new + helper (``decode_view_request_filters()``) to easily decode the new flattened + filter string. (`#6747 `_) +- Add an index on column resource_id in table resource_view. (`#7134 + `_) +- Non-sysadmin users are no longer able to change their own state (`#6956 + `_) +- The "rank" field is no longer returned in datastore_search results unless + explicitly defined in the fields parameter (`#6961 + `_) +- Upgrade requirements to the latest version whenever possible (`#7064 + `_) +- Create a ``fresh_context()`` function to allow cleaning the ``context`` dict + preserving some common values (``user``, ``model``, etc) (`#7112 + `_) +- Add ``--quiet`` option to ``ckan user token add`` command to mak easier to + integrate with automated scripts (`#7217 + `_) +- Updated and documented input param for ``api_token_list`` from ``user`` to + ``user_id``. ``user`` is still supported for backwards compatibility but it might + be removed in the future. (`#7344 `_) + + +Bugfixes +-------- + +- Stable default ordering when consuming resource content from datastore + (`#2317 `_) +- Fix missing activities from UI when internal processes are run by ignored + users (`#5699 `_) +- Fix the datapusher trigger in case of resource_update via API (`#5727 + `_) +- package_revise now returns some errors in normal keys instead of under + 'message' (`#5888 `_) +- Allow multi-level config inheritance (`#6000 + `_) +- Fix Chinese locales. Note that the URLs for the `zh_CN` and `zh_TW` locales + have changed but there are redirects in place, eg + http://localhost:5000/zh_CN/dataset -> + http://localhost:5000/zh_Hans_CN/dataset (`#6008 + `_) +- Fix performance bottleneck in activity queries (`#6028 + `_) +- Keep repeatable facets inside pagination links (`#6084 + `_) +- Consistent CLI behavior when when no command provided and when using `--help` + options (`#6120 `_) +- Variables from extended config files (``use = config:...``) have lower + precedence. + In the following example:: + + ;; a.ini + output = %(var)s + + ;; b.ini + use = config:a.ini + var = B + + ;; c.ini + use = config:b.ini + var = C + + final value of the ``output`` config option will be ``C``. (`#6192 + `_) +- Restore error traceback for `search-index rebuild -i` CLI command (`#6329 + `_) +- Prevent Traceback to logged for HTTP Exception until debug is true + Add the HTTP status Code in logging for HTTP requests (`#6340 + `_) +- Improve rendering data types in resource view (`#6356 + `_) +- Snippet names rendered into HTML as comments in non-debug mode. (`#6406 + `_) +- h.remove_url_param fail with minimal set of params (`#6414 + `_) +- Type of uploads for group and user image can be restricted via the + `ckan.upload.{object_type}.types` and `ckan.upload.{object_type}.mimetypes` + config options (eg `ckan.upload.group.types`, `ckan.upload.user.mimetypes`) + (`#6477 `_) +- ``*_patch`` actions call their ``*_update`` equivalents via ``get_action`` + allowing plugins to override them consistently (`#6519 + `_) +- Fixed and simplified organization and group forms breadcrumb inheritance + (`#6637 `_) +- Ensure that locale exists on i18n JS API (`#6698 + `_) +- Configuration options that were used to specify a CSS file + with a base theme have been removed. Use the altenatives below in order + to specify an _asset_ (see :doc:`theming/webassets`) with a base theme for application + (`#6817 `_): + * ``ckan.main_css`` replaced by :ref:`ckan.theme` + * ``ckan.i18n.rtl_css`` replaced by :ref:`ckan.i18n.rtl_theme` +- prepare_dataset_blueprint: support dataset type (`#7031 + `_) +- Changed default sort key for group and user lists from ASCII Alphebitized to + new `strxfrm` helper, resulting in human-readable alphebitization. (`#7039 + `_) +- Fix resource file size not updating with resource_patch (`#7075 + `_) +- Revert Flask requirement from 2.2.2 to 2.0.3. (`#7082 + `_) +- restore original plugin template directory order after update_config order + change (`#7085 `_) +- Fix urls containing unicode encoded in hex (`#7107 + `_) +- Fix a bug that causes CKAN to only register the first blueprint of plugins. + (`#7108 `_) +- remove old deleted resources on package_update so that performance is + consistent over time (no longer degrading) (`#7119 + `_) +- Beaker session config variables need to be initialised in a newly generated + ckan config file (`#7133 `_) +- Fixed broken organization delete form (`#7150 + `_) +- Fix the current year reference for CKAN documentation (`#7153 + `_) +- Fix bootstrap 3 webassets files to point to valid assets. (`#7161 + `_) +- Fix the display of the License select element in the Dataset form. (`#7162 + `_) +- Build CSS files with latest updates. (`#7163 + `_) +- Fix activity stream icon on Boostrap 5. Migrate activity CSS classes to the + extension folder. (`#7169 `_) +- Fix 404 error when selecting the same date in the changes view (`#7191 + `_) +- Fix display of Popular snippet. Removes old `ckan-icon` scss class. (`#7205 + `_) +- Fix icons and alignment in resource datastore tab. (`#7247 + `_) +- Make heading semantic in bug report template (`#7186 + `_) +- Add title attribute to iframe (`#7187 + `_) +- Fix color contrast in dashboard buttons for web accesibility (`#7193 + `_) +- Make skip to content visible for keyboard-only user (`#7194 + `_) +- Fix color contrast issue in add dataset page (`#7195 + `_) +- Fix color contrast of delete button in user edit page for web accesibility + (`#7199 `_) + +.. _migration-notes-2.10: + +Migration notes +--------------- + +- Changes in the authenticated users management (logged in users): The old ``auth_tkt`` cookie + created by repoze.who does not exist anymore. Flask-login stores the logged-in user + identifier in the Flask session. CKAN uses `Beaker `_ + to manage the session, and the default session backend stores this session information + as files on the server (on ``/tmp``). This means that **if the session data is deleted + in the server, all users will be logged out of the site**. + This can happen for instance: + + * if the CKAN container is redeployed in a Docker / cloud setup and the session directory is not persisted + * if the sessions are periodically cleaned by an external script + + Here's a summary of the behaviour changes between CKAN versions: + + .. list-table:: + :widths: 40 30 30 + :header-rows: 1 + + * - Action + - CKAN < 2.10 + - CKAN >= 2.10 + * - Clear cookies + - User logged out + - User logged out (If ``remember_me`` cookie is deleted) + * - Clear server sessions + - User still logged in + - User logged out + + The way to keep the old behaviour with the Beaker backend is to store the + session data in the `cookie itself `_ + (note that this stores *all* session data, not just the user identifier). This will probably + be the default behaviour in future CKAN versions:: + + # ckan.ini + beaker.session.type = cookie + beaker.session.validate_key = CHANGE_ME + + beaker.session.httponly = True + beaker.session.secure = True + beaker.session.samesite = Lax # or Strict + + Alternatively you can configure another persistent backend for the sessions in the server, + like an SQL Database or Redis (see the `Beaker configuration `_ + for details). +- It is recommended that you review the :ref:`session-settings` and :ref:`flask-login-remember-me-cookie-settings` to + make sure they cover your security requirements. +- Due to the newly introduced :ref:`declare-config-options`, all declared CKAN configuration options + are validated and converted to the expected type during the application startup:: + + debug = config.get("debug") + + # CKAN <= v2.9 + assert type(debug) is str + assert debug == "false" # or any value that is specified in the config file + + # CKAN >= v2.10 + assert type(debug) is bool + assert debug is False # or ``True`` + + The ``aslist``, ``asbool``, ``asint`` converters from + ``ckan.plugins.toolkit`` will keep the current behaviour:: + + # produces the same result in v2.9 and v2.10 + assert tk.asbool(config.get("debug")) is False + assert tk.asint(config.get("ckan.devserver.port")) == 5000 + assert tk.aslist(config.get("ckan.plugins")) == ["stats"] + + If you are using custom logic, the code requires a review. For example, the + following code will produce an ``AttributeError`` exception, because + ``ckan.plugins`` is + converted into a list during the application's startup:: + + # AttributeError + plugins = config.get("ckan.plugins").split() + + Depending on the desired backward compatibility, one of the following + expressions + can be used instead:: + + # if both v2.9 and v2.10 are supported + plugins = tk.aslist(config.get("ckan.plugins")) + + # if only v2.10 is supported + plugins = config.get("ckan.plugins") + + The second major change affects default values for configuration options. + Starting from CKAN 2.10, + the majority of the config options have a declared default value. It means + that + whenever you invoke ``config.get`` method, the *declared default* value is + returned instead of ``None``. Example:: + + # CKAN v2.9 + assert config.get("search.facets.limit") is None + + # CKAN v2.10 + assert config.get("search.facets.limit") == 10 + + The second argument to ``config.get`` should be only used to get + the value of a missing *undeclared* option:: + + assert config.get("not.declared.and.missing.from.config", 1) == 1 + + The above is the same for any extension that *declares* its config options + using ``IConfigDeclaration`` interface or ``config_declarations`` blanket. + (`#6467 `_) +- Public registration of users has been disabled by default (`#7210 + `_) +- User and group/org image upload formats have been restricted by default (`#7210 + `_) +- The activites feature has been extracted into a separate ``activity`` plugin. + To keep showing the activities in the UI and enable the activity related API + actions you need to add the ``activity`` plugin to the :ref:`ckan.plugins` config + option. This change doesn't affect activities already stored in the DB. They are still + available once the plugin is enabled. Note that some imports have changed + (`#6790 `_):: + + `ckan.model.Activity` -> `ckanext.activity.model.Activity` +- Users of the Xloader or DataPusher need to provide a valid API Token in their + configurations using the ``ckanext.xloader.api_token`` or + ``ckan.datapusher.api_token`` keys respectively. (`#7139 + `_) +- Only user-defined functions can be used as validators. An attempt to use + a mock-object, built-in function or class will cause a ``TypeError``. (`#6048 + `_) +- The language code for the Norwegian language has been updated from ``no`` to + ``nb_NO``. There are redirects in place from the old code to the new one for + localized URLs, but please update your links. If you were using the old + ``no`` code in a config option like ``ckan.default_locale`` or + ``ckan.locales_offered`` you will need to update the value to ``nb_NO``. + (`#6746 `_) +- `toolkit.aslist` now converts any iterable other than ``list`` and `tuple` + into a ``list``: ``list(value)``. + Before, such values were just wrapped into a list, i.e: ``[value]`` (`#7257 `_). + + .. list-table:: Short overview of changes + :widths: 40 30 30 + :header-rows: 1 + + * - Expresion + - Before + - After + * - ``aslist([1,2])`` + - ``[1, 2]`` + - ``[1, 2]`` + * - ``aslist({1,2})`` + - ``[{1, 2}]`` + - ``[1, 2]`` + * - ``aslist({1: "one", 2: "two"})`` + - ``[{1: "one", 2: "two"}]`` + - ``[1, 2]`` + * - ``aslist(range(1,3))`` + - ``[range(1, 3)]`` + - ``[1, 2]`` + +Removals and deprecations +------------------------- + +- Legacy API keys are no longer supported for Authentication and have been + removed + from the UI. API Tokens should be used instead. See :ref:`api authentication` + for + more details (`#6247 `_) +- ``build_nav_main()``, ``build_nav_icon()`` and ``build_nav()`` helpers no longer + support + Pylons route syntax. eg use ``dataset.search`` instead of ``controller=dataset, action=search``. + (`#6263 `_) +- The following old helper functions have been removed and are no longer + available: + ``submit()``, ``radio()``, ``icon_url()``, ``icon_html()``, ``icon()``, + ``resource_icon()``, + ``format_icon()``, ``button_attr()``, ``activity_div()`` (`#6272 + `_) +- The following methods are deprecated and should be replaced with their + respective new versions in the plugin interfaces: + + - `ckan.plugins.interfaces.IResourceController`: + + - change ``before_create`` to ``before_resource_create`` + - change ``after_create`` to ``after_resource_create`` + - change ``before_update`` to ``before_resource_update`` + - change ``after_update`` to ``after_resource_update`` + - change ``before_delete`` to ``before_resource_delete`` + - change ``after_delete`` to ``after_resource_delete`` + - change ``before_show`` to ``before_resource_show`` + + - `ckan.plugins.interfaces.IPackageController`: + + - change ``after_create`` to ``after_dataset_create`` + - change ``after_update`` to ``after_dataset_update`` + - change ``after_delete`` to ``after_dataset_delete`` + - change ``after_show`` to ``after_dataset_show`` + - change ``before_search`` to ``before_dataset_search`` + - change ``after_search`` to ``after_dataset_search`` + - change ``before_index`` to ``before_dataset_index`` + + | (`#6501 `_) +- The ``ckan seed`` command has been removed in favour of ``ckan generate + fake-data`` + for generating test entities in the database. Refer to ``ckan generate + fake-data --help`` + for some usage examples. (`#6504 `_) +- The ``IRoutes`` interface has been removed since it was part of the old Pylons + architecture. (`#6594 `_) +- Remove ``ckan.cache_validated_datasets`` config (`#6628 + `_) +- Remove ``ckan.search.automatic_indexing`` config (`#6639 + `_) +- The ``PluginMapperExtension`` has been removed since it was no longer used in + core + and it had a deprecated dependency. (`#6648 + `_) +- Remove deprecated ``fields`` parameter in ``resource_search`` method. (`#6687 + `_) +- The ``ISession`` interface has been removed from CKAN. To extend SQLAlchemy use + event listeners instead. (`#6699 `_) +- ``unselected_facet_items`` helper has been removed. You can use + ``get_facet_items_dict`` with ``exclude_active=True`` instead. (`#6765 + `_) +- The Recline-based view plugins (``recline_view``, ``recline_grid_view``, + ``recline_graph_view`` and ``recline_map_view``) are deprecated and will be + removed in future versions. Check :doc:`maintaining/data-viewer` for alternatives. + (`#7078 `_) +- The requirement-setuptools.txt file has been removed (`#7271 `_) +- ``ckan.route_after_login`` renamed to ``ckan.auth.route_after_login`` (`#7350 + `_) + + +v.2.9.7 2022-10-26 +================== + +Bugfixes +-------- + +* CVE-2022-43685: fix potential user account takeover via user create +* Fix Datatables view download format selector (`#7147 `_) +* Revert deletions included in 2.9.6 as part of #6187 (`#7118 `_) + + +v.2.9.6 2022-09-28 +================== + +Note: This release includes requirements upgrades to address security issues + + +Bugfixes +-------- + +- Fixes incorrectly encoded url current_url (`#6685 `_) +- Check if locale exists on i18n JS API (`#6698 `_) +- Add ``csrf_input()`` helper for cross-CKAN version compatibilty (`#7016 `_) +- Fix not empty validator (`#6658 `_) +- Use ``get_action()`` in patch actions to allow custom logic (`#6519 `_) +- Allow to extend organization_facets (`#6682 `_) +- Expose check_ckan_version to templates (`#6741 `_) +- Allow get_translated helper to fall back to base version of a language (`#6815 `_) +- Fix server error in tag autocomplete when vocabulary does not exist (`#6820 `_) +- Check if locale exists on i18n JS API (`#6698 `_) +- Fix updating a non-existing resource causes an internal sever error (`#6928 `_) +- Remove extra comma (`#6774 `_) +- Fix test data creation issues (`#6805 `_) +- Fix for updating non-existing resource +- Avoid storing the session on each request (`#6954 `_) +- Return zero results instead of raising NotFound when vocabulary does not exist +- Fix the datapusher trigger in case of resource_update via API (`#5727 `_) +- Consistent CLI behavior when when no command provided and when using `--help` options (`#6120 `_) +- Fix regression when validating resource subfields (`#6546 `_) +- Fix resource file size not updating with resource_patch (`#7075 `_) +- Prevent non-sysadmin users to change their own state (`#6956 `_) +- Use user id in auth cookie rather than name +- Reorder resource view button: allow translation (`#6089 `_) +- Optmize temp dir creation on uploads (`#6578 `_) +- Exclude site_user from user_listi (`#6618 `_) +- Fix race condition in creating the default site user (`#6638 `_) +- gettext not for metadata fields (`#6660 `_) +- Include root_path in activity email notifications (`#6743 `_) +- Extract translations from emails (`#5857 `_) +- Use the headers Reply-to value if its set in the extensions (`#6838 `_) +- Improve error when downloading resource (`#6832 `_) +- ``ckan_config`` test mark works with request context (`#6868 `_) +- Fix caching logic on logged in users (`#6864 `_) +- Fix member delete (`#6892 `_) +- Concurrent-safe resource updates (`#6439 `_) +- Fix error when listing tokens in the CLI in py2 (`#6789 `_) + +Minor changes +------------- + +- The ``ckan.main_css`` and ``ckan.i18.rtl_css`` settings, which were not working, have been replaced by :ref:`ckan.theme` and :ref:`ckan.i18n.rtl_theme` respectively. Both expect the name of an *asset* with a base theme for the application (`#6817 `_) +- The type of uploads for group and user image can be restricted via the `ckan.upload.{object_type}.types` and `ckan.upload.{object_type}.mimetypes` config options (eg :ref:`ckan.upload.group.types`, :ref:`ckan.upload.user.mimetypes`) (`#6477 `_) +- Allow to use PDB and IDE debuggers (`#6798 `_) +- Unpin pytz, upgrade zope.interface (`#6665 `_) +- Update sqlparse version +- Bump markdown requirement to support Python 3.9 +- Update psycopg2 to support PostgreSQL 12 +- Add auth functions for 17 actions that didn't have them before (`#7045 `_) +- Add no-op ``csrf_input()`` helper to help extensions with cross-CKAN version suport (`#7030 `_) + + +v.2.9.5 2022-01-19 +================== + + +Major features +-------------- + +- Solr 8 support. Starting from version 2.9.5, CKAN supports Solr versions 6 and 8. Support for Solr 6 will be dropped in the next + CKAN minor version (2.10). Note that if you want to use Solr 8 you need to use the ``ckan/config/solr/schema.solr8.xml`` file, or + alternatively you can use the ``ckan/ckan-solr:2.9-solr8`` Docker image which comes pre-configured. (`#6530 `_) + + +Bugfixes +-------- + +- Consistent CLI behavior when no command is provided and when using `--help` (`#6120 `_) +- Fix regression when validating resource subfields (`#6546 `_) +- Fix user create/edit email validators (`#6399 `_) +- Error opening JS translations on Python 2 (`#6531 `_) +- Set logging level to error in error mail handler (`#6577 `_) +- Add RootPathMiddleware to flask stack to support non-root installs running on python 3 (`#6556 `_) +- Use correct auth function when editing organizations (`#6622 `_) +- Fix invite user with existing email error (`#5880 `_) +- Accept empty string in one of validator (`#6612 `_) + + +Minor changes +------------- + +- Add timeouts to requests calls (see `ckan.requests.timeout`) (`#6408 `_) +- Types of file uploads for group and user imags can be restricted via the `ckan.upload.{object_type}.types` and `ckan.upload.{object_type}.mimetypes` config options (eg :ref:`ckan.upload.group.types`, :ref:`ckan.upload.user.mimetypes`) (`#6477 `_) +- Allow children elements on select2 lists (`#6503 `_) +- Enable ``minimumInputLength`` and fix loading message in select2 (`#6554 `_) + + +v.2.9.4 2021-09-22 +================== + +Note: This release includes requirements upgrades to address security issues + + +Bugfixes +-------- + +- Don't show snippet names in non-debug mode (`#6406 `_) +- Show job title on job start/finish log messages (`#6387 `_) +- Fix unpriviledged users being able to access bulk process (`#6290 `_) +- Allow UTF-8 in JS translations (`#6051 `_) +- Handle Traceback Exception for HTTP and HTTP status Code in logging (`#6340 `_) +- Fix object list validation output (`#6149 `_) +- Coerce query string keys/values before passing to quote() (`#6099 `_) +- Fix datetime formatting when listing user tokens on py2. (`#6319 `_) +- Fix Solr HTTP basic auth cred handling (`#6286 `_) +- Remove not accessed user object in resource_update (`#6220 `_) +- Fix for g.__timer (`#6207 `_) +- Fix guard clause on has_more_facets, #6190 (`#6190 `_) +- Fix page render errors when search facets are not defined (`#6181 `_) +- Fix exception when using solr_user and solr_password on Py3 (`#6179 `_) +- Fix pagination links for custom org types (`#6162 `_) +- Fixture for plugin DB migrations (`#6139 `_) +- Render activity timestamps with title= attribute (`#6109 `_) +- Fix db init error in alembic (`#5998 `_) +- Fix user email validator when using name as id parameter (`#6113 `_) +- Fix DataPusher error during resource_update (`#5597 `_) +- render_datetime helper does not respect ckan.display_timezone configuration (`#6252 `_) +- Fix SQLAlchemy configuration for DataStore (`#6087 `_) +- Don't cache license translations across requests (`#5586 `_) +- Fix tracking.js module preventing links to be opened in new tabs (`#6386 `_) +- Fix deleted org/group feeds (`#6368 `_) +- Fix runaway preview height (`#6284 `_) +- Stable default ordering when consuming resource content from datastore + (`#2317 `_) +- Several documentation fixes and improvements + +v.2.9.3 2021-05-19 +================== + +Bugfixes +-------- + +- Fix Chinese locales. Note that the URLs for the `zh_CN` and `zh_TW` locales + have changed but there are redirects in place, eg + http://localhost:5000/zh_CN/dataset -> + http://localhost:5000/zh_Hans_CN/dataset (`#6008 + `_) +- Fix performance bottleneck in activity queries (`#6028 + `_) +- Keep repeatable facets inside pagination links (`#6084 + `_) +- Ensure order of plugins in PluginImplementations (`#5965 `_) +- Fix for Datastore file dump extension (`#5593 `_) +- Allow package activity migration on py3 (`#5930 `_) +- Fix TemplateSyntaxError in snippets/changes/license.html (`#5972 `_) +- Remove hardcoded logging level (`#5941 `_) +- Include extra files into ckanext distribution (`#5995 `_) +- Fix db init in docker as the directory is not empty (`#6027 `_) +- Fix sqlalchemy configuration, add doc (`#5932 `_) +- Fix issue with purging custom entity types (`#5859 `_) +- Only load view filters on templates that need them +- Sanitize user image url +- Allow installation of requirements without any additional actions using pip (`#5408 `_) +- Include requirements files in Manifest (`#5726 `_) +- Dockerfile: pin pip version (`#5929 `_) +- Allow uploaders to only override asset / resource uploading (`#6088 `_) +- Catch TypeError from invalid thrown by dateutils (`#6085 `_) +- Display proper message when sysadmin password is incorect (`#5911 `_) +- Use external library to parse view filter params +- Fix auth error when deleting a group/org (`#6006 `_) +- Fix datastore_search language parameter (`#5974 `_) +- make SQL function whitelist case-insensitive unless quoted (`#5969 `_) +- Fix Explore button not working (`#3720 `_) +- remove unused var in task_status_update (`#5861 `_) +- Prevent guessing format and mimetype from resource urls without path (`#5852 `_) +- Multiple documentation improvements + + +Minor changes +------------- + +- Support for setting host and port on the ini file (`#5939 `_) +- Allow to set path to INI file in the WSGI script (`#5987 `_) +- Allow multi-level config inheritance (`#6000 + `_) + + +v.2.9.2 2021-02-10 +================== + +General notes: + * Note: To use PostgreSQL 12 on CKAN 2.9 you need to upgrade psycopg2 to at least 2.8.4 (more details in `#5796 `_) + + +Major features +-------------- + +- Add CLI commands for API Token management (`#5868 + `_) + + +Bugfixes +-------- + +- Persist attributes in chained functions (`#5751 `_) +- Fix install documentation (`#5618 `_) +- Fix exception when passing limit to organization (`#5789 `_) +- Fix for adding directories from plugins if partially string matches existing values (`#5836 `_) +- Fix upload log activity sorting (`#5827 `_) +- Textview: escape text formats (`#5814 `_) +- Add allow_partial_update to fix losing users (`#5734 `_) +- Set default group_type to group in group_create (`#5693 `_) +- Use user performing the action on activity context on user_update (`#5743 `_) +- New block in nav links in user dashboard (`#5804 `_) +- Update references to DataPusher documentation +- Fix JavaScript error on Edge (`#5782 `_) +- Fix error when deleting resource with missing datastore table (`#5757 `_) +- ensure HTTP_HOST is bytes under python2 (`#5714 `_) +- Don't set old_filename when updating groups (`#5707 `_) +- Filter activities from user at the database level (`#5698 `_) +- Fix user_list ordering (`#5667 `_) +- Allowlist for functions in datastore_search_sql (see :ref:`ckan.datastore.sqlsearch.allowed_functions_file`) +- Fix docker install (`#5381 `_) +- Fix Click requirement conflict (`#5539 + `_) +- Return content-type header on downloads if mimetype is (`#5670 + `_) +- Fix missing activities from UI when internal processes are run by ignored + users (`#5699 `_) +- Replace 'paster' occurrences with 'ckan' in docs (`#5700 + `_) +- Include requirements files in Manifest (`#5726 + `_) +- Fix order which plugins are returned by PluginImplementations changing + (`#5731 `_) +- Raise NotFound when creating a non-existing collaborator (`#5759 + `_) +- Restore member edit page (`#5767 `_) +- Don't add --ckan-ini pytest option if already added (by pytest-ckan) (`#5774 + `_) +- Update organization_show package limit docs (`#5784 + `_) +- Solve encoding errors in changes templates (`#5785 + `_) + + +Minor changes +------------- + +- Add aria attribute and accessible screen reader text to the mobile nav + button. (`#5555 `_) +- Remove jinja2 blocks from robots.txt (`#5648 + `_) +- Allow to run the development server using SSL (`#5825 + `_) +- Update extension template, migrate tests to GitHub Actions (`#5797 + `_) + + v.2.9.1 2020-10-21 ================== @@ -115,8 +895,8 @@ Migration notes migrate_package_activity.py like this:: cd /usr/lib/ckan/default/src/ckan/ - wget https://raw.githubusercontent.com/ckan/ckan/3484_revision_ui_removal2/ckan/migration/migrate_package_activity.py - wget https://raw.githubusercontent.com/ckan/ckan/3484_revision_ui_removal2/ckan/migration/revision_legacy_code.py + wget https://raw.githubusercontent.com/ckan/ckan/2.9/ckan/migration/migrate_package_activity.py + wget https://raw.githubusercontent.com/ckan/ckan/2.9/ckan/migration/revision_legacy_code.py python migrate_package_activity.py -c /etc/ckan/production.ini Future versions of CKAN are likely to need a slightly different procedure. @@ -320,6 +1100,104 @@ Removals and deprecations (`#5112 `_) - Remove paster CLI (`#5264 `_) +v.2.8.12 2022-10-26 +=================== + +Bugfixes +-------- + +* CVE-2022-43685: fix potential user account takeover via user create + +v.2.8.11 2022-09-28 +=================== + +Fixes: + +* Fixes incorrectly encoded url current_url (`#6685 `_) +* Check if locale exists on i18n JS API (`#6698 `_) +* Add ``csrf_input()`` helper for cross-CKAN version compatibilty (`#7016 `_) +* Fix not empty validator (`#6658 `_) +* Use ``get_action()`` in patch actions to allow custom logic (`#6519 `_) +* Allow to extend organization_facets (`#6682 `_) +* Expose check_ckan_version to templates (`#6741 `_) +* Allow get_translated helper to fall back to base version of a language (`#6815 `_) +* Fix server error in tag autocomplete when vocabulary does not exist (`#6820 `_) +* Check if locale exists on i18n JS API (`#6698 `_) +* Fix updating a non-existing resource causes an internal sever error (`#6928 `_) + + +v.2.8.10 2022-01-19 +=================== + +Fixes: + +* Add timeouts to requests calls (see `ckan.requests.timeout`) (`#6408 `_) +* Fix user create/edit email validators (`#6399 `_) +* Allow children elements on select2 lists (`#6503 `_) + + + +v.2.8.9 2021-09-22 +================== + +Fixes: + +* render_datetime helper does not respect ckan.display_timezone configuration (`#6252 `_) +* Fix SQLAlchemy configuration for DataStore (`#6087 `_) +* Don't cache license translations across requests (`#5586 `_) +* Fix tracking.js module preventing links to be opened in new tabs (`#6386 `_) +* Fix deleted org/group feeds (`#6368 `_) +* Fix runaway preview height (`#6284 `_) +* Fix unreliable ordering of DataStore results (`#2317 `_) + + +v.2.8.8 2021-05-19 +================== + +* Fix Chinese locales (`#4413 `_) +* Allow installation of requirements without any additional actions using pip (`#5408 `_) +* Include requirements files in Manifest (`#5726 `_) +* Dockerfile: pin pip version (`#5929 `_) +* Allow uploaders to only override asset / resource uploading (`#6088 `_) +* Catch TypeError from invalid thrown by dateutils (`#6085 `_) +* Display proper message when sysadmin password is incorect (`#5911 `_) +* Use external library to parse view filter params +* Fix auth error when deleting a group/org (`#6006 `_) +* Fix datastore_search language parameter (`#5974 `_) +* make SQL function whitelist case-insensitive unless quoted (`#5969 `_) +* Fix Explore button not working (`#3720 `_) +* remove unused var in task_status_update (`#5861 `_) +* Prevent guessing format and mimetype from resource urls without path (`#5852 `_) + +v.2.8.7 2021-02-10 +================== + +General notes: +* Note: To use PostgreSQL 12 on CKAN 2.8 you need to upgrade SQLAlchemy to 1.2.17 and vdm to 0.15 (more details in `#5796 `_) + + +Fixes: + +* Persist attributes in chained functions (`#5751 `_) +* Fix install documentation (`#5618 `_) +* Fix exception when passing limit to organization (`#5789 `_) +* Fix for adding directories from plugins if partially string matches existing values (`#5836 `_) +* Fix upload log activity sorting (`#5827 `_) +* Textview: escape text formats (`#5814 `_) +* Add allow_partial_update to fix losing users (`#5734 `_) +* Set default group_type to group in group_create (`#5693 `_) +* Use user performing the action on activity context on user_update (`#5743 `_) +* New block in nav links in user dashboard (`#5804 `_) +* Update references to DataPusher documentation +* Fix JavaScript error on Edge (`#5782 `_) +* Fix error when deleting resource with missing datastore table (`#5757 `_) +* ensure HTTP_HOST is bytes under python2 (`#5714 `_) +* Don't set old_filename when updating groups (`#5707 `_) +* Filter activities from user at the database level (`#5698 `_) +* Fix user_list ordering (`#5667 `_) +* Allowlist for functions in datastore_search_sql (see :ref:`ckan.datastore.sqlsearch.allowed_functions_file`) + + v.2.8.6 2020-10-21 ================== @@ -367,7 +1245,7 @@ General notes: * Note: This version does not require a Solr schema upgrade * Note: This version includes changes in the way the ``SameSite`` flag is set on the ``auth_tkt`` authorization cookie. The new default setting for it is ``SameSite=Lax``, which aligns with the behaviour of all major browsers. If for some - reason you need a different value, you can set it via the :ref:`who.samesite` configuration option. You can find more + reason you need a different value, you can set it via the `who.samesite` configuration option. You can find more information on the ``SameSite`` attribute `here `_. @@ -603,6 +1481,56 @@ Changes and deprecations: to specify this argument explicitly, as in future it'll become required. * The ``ckan.recaptcha.version`` config option is now removed, since v2 is the only valid version now (#4061) +v.2.7.12 2021-09-22 +=================== + +Fixes: + +* Fix tracking.js module preventing links to be opened in new tabs (`#6384 `_) +* Fix deleted org/group feeds (`#6367 `_) +* Fix runaway preview height (`#6283 `_) +* Fix unreliable ordering of DataStore results (`#2317 `_) + +v.2.7.11 2021-05-19 +=================== + +Fixes: + +* Allow uploaders to only override asset / resource uploading (`#6088 `_) +* Catch TypeError from invalid thrown by dateutils (`#6085 `_) +* Use external library to parse view filter params +* Fix auth error when deleting a group/org (`#6006 `_) +* Fix datastore_search language parameter (`#5974 `_) +* make SQL function whitelist case-insensitive unless quoted (`#5969 `_) +* Fix Explore button not working (`#3720 `_) +* "New view" button fix (`#4260 `_) +* remove unused var in task_status_update (`#5861 `_) +* Prevent guessing format and mimetype from resource urls without path (`#5852 `_) + +v.2.7.10 2021-02-10 +=================== + +Fixes: + +* Fix install documentation (`#5618 `_) +* Fix exception when passing limit to organization (`#5789 `_) +* Fix for adding directories from plugins if partially string matches existing values (`#5836 `_) +* Fix upload log activity sorting (`#5827 `_) +* Textview: escape text formats (`#5814 `_) +* Add allow_partial_update to fix losing users (`#5734 `_) +* Set default group_type to group in group_create (`#5693 `_) +* Use user performing the action on activity context on user_update (`#5743 `_) +* New block in nav links in user dashboard (`#5804 `_) +* Update references to DataPusher documentation +* Fix JavaScript error on Edge (`#5782 `_) +* Fix error when deleting resource with missing datastore table (`#5757 `_) +* ensure HTTP_HOST is bytes under python2 (`#5714 `_) +* Don't set old_filename when updating groups (`#5707 `_) +* Filter activities from user at the database level (`#5698 `_) +* Fix user_list ordering (`#5667 `_) +* Allow list for functions in datastore_search_sql (see :ref:`ckan.datastore.sqlsearch.allowed_functions_file`) + + v.2.7.9 2020-10-21 ================== @@ -650,7 +1578,7 @@ General notes: * Note: This version does not require a Solr schema upgrade * Note: This version includes changes in the way the ``SameSite`` flag is set on the ``auth_tkt`` authorization cookie. The new default setting for it is ``SameSite=Lax``, which aligns with the behaviour of all major browsers. If for some - reason you need a different value, you can set it via the :ref:`who.samesite` configuration option. You can find more + reason you need a different value, you can set it via the `who.samesite` configuration option. You can find more information on the ``SameSite`` attribute `here `_. @@ -1470,7 +2398,7 @@ Changes and deprecations * The default of allowing anyone to create datasets, groups and organizations has been changed to False. It is advised to ensure you set all of the - :ref:`config-authorization` options explicitly in your CKAN config. (#2164) + :ref:`authorization-settings` options explicitly in your CKAN config. (#2164) * The ``package_show`` API call does not return the ``tracking_summary``, keys in the dataset or resources by default any more. @@ -1784,7 +2712,7 @@ Changes and deprecations * The HttpOnly flag will be set on the authorization cookie by default. For enhanced security, we recommend using the HttpOnly flag, but this behaviour can be changed in the ``Repoze.who`` settings detailed in the Config File - Options documentation (:ref:`who.httponly`). + Options documentation (`who.httponly`). * The OpenID login option has been removed and is no longer supported. See "Troubleshooting" if you are upgrading an existing CKAN instance as you may @@ -2419,7 +3347,7 @@ New frontend (see :doc:`/theming/index`): CKAN's frontend has been completely redesigned, inside and out. There is a new default theme and the template engine has moved from Genshi to Jinja2. Any custom templates using Genshi will need to be updated, although - there is a :ref:`ckan.legacy_templates` setting to aid in the migration. + there is a ``ckan.legacy_templates`` setting to aid in the migration. * Block-based template inheritance * Custom jinja tags: {% ckan_extends %}, {% snippet %} and {% url_for %} (#2502, #2503) diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 87164a78983..00000000000 --- a/Dockerfile +++ /dev/null @@ -1,66 +0,0 @@ -# See CKAN docs on installation from Docker Compose on usage -FROM debian:stretch -MAINTAINER Open Knowledge - -# Install required system packages -RUN apt-get -q -y update \ - && DEBIAN_FRONTEND=noninteractive apt-get -q -y upgrade \ - && apt-get -q -y install \ - python-dev \ - python-pip \ - python-virtualenv \ - python-wheel \ - python3-dev \ - python3-pip \ - python3-virtualenv \ - python3-wheel \ - libpq-dev \ - libxml2-dev \ - libxslt-dev \ - libgeos-dev \ - libssl-dev \ - libffi-dev \ - postgresql-client \ - build-essential \ - git-core \ - vim \ - wget \ - && apt-get -q clean \ - && rm -rf /var/lib/apt/lists/* - -# Define environment variables -ENV CKAN_HOME /usr/lib/ckan -ENV CKAN_VENV $CKAN_HOME/venv -ENV CKAN_CONFIG /etc/ckan -ENV CKAN_STORAGE_PATH=/var/lib/ckan - -# Build-time variables specified by docker-compose.yml / .env -ARG CKAN_SITE_URL - -# Create ckan user -RUN useradd -r -u 900 -m -c "ckan account" -d $CKAN_HOME -s /bin/false ckan - -# Setup virtual environment for CKAN -RUN mkdir -p $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH && \ - virtualenv $CKAN_VENV && \ - ln -s $CKAN_VENV/bin/pip /usr/local/bin/ckan-pip &&\ - ln -s $CKAN_VENV/bin/paster /usr/local/bin/ckan-paster &&\ - ln -s $CKAN_VENV/bin/ckan /usr/local/bin/ckan - -# Setup CKAN -ADD . $CKAN_VENV/src/ckan/ -RUN ckan-pip install -U pip && \ - ckan-pip install --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/requirement-setuptools.txt && \ - ckan-pip install --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/requirements-py2.txt && \ - ckan-pip install -e $CKAN_VENV/src/ckan/ && \ - ln -s $CKAN_VENV/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini && \ - cp -v $CKAN_VENV/src/ckan/contrib/docker/ckan-entrypoint.sh /ckan-entrypoint.sh && \ - chmod +x /ckan-entrypoint.sh && \ - chown -R ckan:ckan $CKAN_HOME $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH - -ENTRYPOINT ["/ckan-entrypoint.sh"] - -USER ckan -EXPOSE 5000 - -CMD ["ckan","-c","/etc/ckan/production.ini", "run", "--host", "0.0.0.0"] diff --git a/LICENSE.txt b/LICENSE.txt index 5385316e7c1..4dfb2cdcb99 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -71,28 +71,3 @@ Permission is hereby granted, free of charge, to any person obtaining a copy of The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -rjsmin / rcssmin ----------------- - -Parts of these packages are include in the include directory of ckan. -Full packages can be found at. -http://opensource.perlig.de/rjsmin/ -http://opensource.perlig.de/rcssmin/ - -They both are licensed under Approved License, Version 2 - -Copyright 2011, 2012 -Andr\xe9 Malo or his licensors, as applicable - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/MANIFEST.in b/MANIFEST.in index 9e4e5a17a6a..b24accfdc68 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,25 +1,29 @@ include ckan/config/deployment.ini_tmpl +recursive-include ckan *.py recursive-include ckan/public * +recursive-include ckan/public-bs3 * recursive-include ckan/config *.ini recursive-include ckan/config *.json +recursive-include ckan/config *.yaml recursive-include ckan/config *.xml recursive-include ckan/i18n * recursive-include ckan/templates * +recursive-include ckan/templates-bs3 * recursive-include ckan *.ini +recursive-include ckanext *.py recursive-include ckanext/*/i18n * recursive-include ckanext/*/public * +recursive-include ckanext/*/assets * recursive-include ckanext/*/templates * recursive-include ckanext/*/theme/public * recursive-include ckanext/*/theme/templates * +recursive-include ckanext/* config_declaration.yaml include ckanext/datastore/set_permissions.sql +include ckanext/datastore/allowed_functions.txt prune .git include CHANGELOG.txt -include ckan/migration/migrate.cfg include ckan/migration/README -recursive-include ckan/migration/versions *.sql -include requirement-setuptools.txt include requirements.txt -include requirements-py2.txt include dev-requirements.txt diff --git a/README.rst b/README.rst index 02fa837dd28..6a728ddbb45 100644 --- a/README.rst +++ b/README.rst @@ -51,7 +51,7 @@ If you find a potential security vulnerability please email security@ckan.org, rather than creating a public issue on GitHub. .. _CKAN tag on Stack Overflow: http://stackoverflow.com/questions/tagged/ckan -.. _archives: https://www.google.com/search?q=%22%5Bckan-dev%5D%22+site%3Alists.okfn.org. +.. _archives: https://groups.google.com/a/ckan.org/g/ckan-dev .. _GitHub Issues: https://github.com/ckan/ckan/issues .. _CKAN chat on Gitter: https://gitter.im/ckan/chat @@ -60,7 +60,7 @@ Contributing to CKAN -------------------- For contributing to CKAN or its documentation, see -`CONTRIBUTING `_. +`CONTRIBUTING `_. Mailing List ~~~~~~~~~~~~ diff --git a/SECURITY.md b/SECURITY.md index bed9f219ede..fb2228be2b5 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,7 +4,7 @@ Security updates are offered for the **3 most recent** minor CKAN releases. They are included in patch releases, but not acknowledged in the release announcement or [CHANGELOG.rst](CHANGELOG.rst), hence the advice to always run the latest patch releases. -For more about CKAN releases see: https://docs.ckan.org/en/2.8/maintaining/upgrading/#ckan-releases +For more about CKAN releases see: http://docs.ckan.org/en/latest/maintaining/upgrading/index.html#ckan-releases ## Reporting a Vulnerability diff --git a/bin/travis-install-dependencies b/bin/travis-install-dependencies deleted file mode 100755 index 0658a7be0be..00000000000 --- a/bin/travis-install-dependencies +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -# Exit immediately if any command fails -set -e - -# Drop Travis' postgres cluster if we're building using a different pg version -TRAVIS_PGVERSION='9.1' -if [ $PGVERSION != $TRAVIS_PGVERSION ] -then - sudo -u postgres pg_dropcluster --stop $TRAVIS_PGVERSION main - # Make psql use $PGVERSION - export PGCLUSTER=$PGVERSION/main -fi - -# Install postgres and solr -sudo apt-get update -qq -sudo apt-get install postgresql-$PGVERSION solr-jetty libcommons-fileupload-java:amd64=1.2.2-1 - -if [ $PGVERSION == '8.4' ] -then - # force postgres to use 5432 as it's port - sudo sed -i -e 's/port = 5433/port = 5432/g' /etc/postgresql/8.4/main/postgresql.conf -fi - -sudo service postgresql restart - -# Setup postgres' users and databases -sudo -E -u postgres ./bin/postgres_init/1_create_ckan_db.sh -sudo -E -u postgres ./bin/postgres_init/2_create_ckan_datastore_db.sh - -export PIP_USE_MIRRORS=true -pip install -r requirement-setuptools.txt --allow-all-external -pip install -r requirements-py2.txt --allow-all-external -pip install -r dev-requirements.txt --allow-all-external - -python setup.py develop - -# Install npm deps -npm install - -paster db init -c test-core.ini - -# If Postgres >= 9.0, we don't need to use datastore's legacy mode. -if [ $PGVERSION != '8.4' ] -then - sed -i -e 's/.*datastore.read_url.*/ckan.datastore.read_url = postgresql:\/\/datastore_default:pass@\/datastore_test/' test-core.ini - paster datastore -c test-core.ini set-permissions | sudo -u postgres psql -else - sed -i -e 's/.*datastore.read_url.*//' test-core.ini -fi - -cat test-core.ini diff --git a/bin/travis-run-tests b/bin/travis-run-tests deleted file mode 100755 index ce309d9eee5..00000000000 --- a/bin/travis-run-tests +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh - -# Configure Solr -echo "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty -sudo cp ckan/config/solr/schema.xml /etc/solr/conf/schema.xml -sudo service jetty restart - -# Run mocha front-end tests -# We need ckan to be running for some tests -paster serve test-core.ini & -sleep 5 # Make sure the server has fully started -npx cypress run -# Did an error occur? -CYPRESS_ERROR=$? -# We are done so kill ckan -killall paster - -# And finally, run the tests -PYTEST_OPTIONS: -v --ckan-ini=test-core.ini --cov=ckan --cov=ckanext --junitxml=/root/junit/junit.xml -python -m pytest $PYTEST_COMMON_OPTIONS -# Did an error occur? -PYTEST_ERROR=$? - -[ "0" -ne "$MOCHA_ERROR" ] && echo MOCHA tests have failed -[ "0" -ne "$PYTEST_ERROR" ] && echo PYTEST tests have failed - -# If an error occurred in our tests make sure travis knows -exit `expr $CYPRESS_ERROR + $PYTEST_ERROR` diff --git a/changes/2317.bugfix b/changes/2317.bugfix new file mode 100644 index 00000000000..a08025755e5 --- /dev/null +++ b/changes/2317.bugfix @@ -0,0 +1 @@ +Stable default ordering when consuming resource content from datastore diff --git a/changes/2628.feature b/changes/2628.feature new file mode 100644 index 00000000000..c713f7cf790 --- /dev/null +++ b/changes/2628.feature @@ -0,0 +1 @@ +Add `organization_followee_count` to the get api \ No newline at end of file diff --git a/changes/3972.migration b/changes/3972.migration deleted file mode 100644 index 17eaa7f5a4f..00000000000 --- a/changes/3972.migration +++ /dev/null @@ -1,6 +0,0 @@ -A full history of dataset changes is now displayed in the Activity Stream to -admins, and optionally to the public. By default this is enabled for new -installs, but disabled for sites which upgrade (just in case the history is -sensitive). When upgrading, open data CKANs are encouraged to make this -history open to the public, by setting this in production.ini: -``ckan.auth.public_activity_stream_detail = true`` diff --git a/changes/3972.removal b/changes/3972.removal deleted file mode 100644 index 697e0967263..00000000000 --- a/changes/3972.removal +++ /dev/null @@ -1,2 +0,0 @@ -Revision and History UI is removed: `/revision/*` & `/dataset/{id}/history` in favour of `/dataset/changes/` visible in the Activity Stream. -``model.ActivityDetail`` is no longer used and will be removed in the next CKAN release. diff --git a/changes/4130.bugfix b/changes/4130.bugfix deleted file mode 100644 index f209d902389..00000000000 --- a/changes/4130.bugfix +++ /dev/null @@ -1 +0,0 @@ -500 error when calling `resource_search` by `last_modified` diff --git a/changes/4319.removal b/changes/4319.removal deleted file mode 100644 index 9d0e3b68cf7..00000000000 --- a/changes/4319.removal +++ /dev/null @@ -1,16 +0,0 @@ -``c.action`` and ``c.controller`` variables should be avoided. -``ckan.plugins.toolkit.get_endpoint`` can be used instead. This function -returns tuple of two items(depending on request handler): -1. Flask blueprint name / Pylons controller name -2. Flask view name / Pylons action name -In some cases, Flask blueprints have names that are differs from their -Pylons equivalents. For example, 'package' controller is divided between -'dataset' and 'resource' blueprints. For such cases you may need to perform -additional check of returned value: - ->>> if toolkit.get_endpoint()[0] in ['dataset', 'package']: ->>> do_something() - -In this code snippet, will be called if current request is handled via Flask's -dataset blueprint in CKAN>=2.9, and, in the same time, it's still working for -Pylons package controller in CKAN<2.9 diff --git a/changes/4448.bugfix b/changes/4448.bugfix deleted file mode 100644 index 7be32ccc0cf..00000000000 --- a/changes/4448.bugfix +++ /dev/null @@ -1 +0,0 @@ -Action function "datastore_search" would calculate the total, even if you set ``include_total=False``. diff --git a/changes/4448.misc b/changes/4448.misc deleted file mode 100644 index 9fc2165c2b1..00000000000 --- a/changes/4448.misc +++ /dev/null @@ -1,2 +0,0 @@ -For navl schemas, the 'default' validator no longer applies the default when -the value is False, 0, [] or {} diff --git a/changes/4450.misc b/changes/4450.misc deleted file mode 100644 index f45f1f52dfb..00000000000 --- a/changes/4450.misc +++ /dev/null @@ -1 +0,0 @@ -Use alembic instead of sqlalchemy-migrate for managing database migrations diff --git a/changes/4484.misc b/changes/4484.misc deleted file mode 100644 index 357aaa53e96..00000000000 --- a/changes/4484.misc +++ /dev/null @@ -1,2 +0,0 @@ -If you've customized the schema for package_search, you'll need to add to it -the limiting of ``row``, as per default_package_search_schema now does. diff --git a/changes/4562.misc b/changes/4562.misc deleted file mode 100644 index 0370eb97f7b..00000000000 --- a/changes/4562.misc +++ /dev/null @@ -1,4 +0,0 @@ -Several logic functions now have new upper limits to how many items can be -returned, notably ``group_list``, ``organization_list`` when -``all_fields=true``, ``datastore_search`` and ``datastore_search_sql``. -These are all configurable. diff --git a/changes/4614.migration b/changes/4614.migration deleted file mode 100644 index 044da01189f..00000000000 --- a/changes/4614.migration +++ /dev/null @@ -1,5 +0,0 @@ -All the css/js files must be bundled via mandatory -`webassets.yml`. Previously, optional `resource.config` was used for -bundling. Check `Assets documentation -`_ -for detailed explanation. diff --git a/changes/4614.misc b/changes/4614.misc deleted file mode 100644 index 24735515cd6..00000000000 --- a/changes/4614.misc +++ /dev/null @@ -1 +0,0 @@ -Replace `fanstatic `_ with `webassets `_ diff --git a/changes/4618.feature b/changes/4618.feature deleted file mode 100644 index 3460c9ab7f8..00000000000 --- a/changes/4618.feature +++ /dev/null @@ -1,4 +0,0 @@ -Safe dataset updates with ``package_revise``: This is a new API action for safe concurrent changes -to datasets and resources. ``package_revise`` allows assertions about current package metadata, -selective update and removal of fields at any level, and multiple file uploads in a single call. -See the documentation at :py:func:`~ckan.logic.action.update.package_revise` \ No newline at end of file diff --git a/changes/4627.removal b/changes/4627.removal deleted file mode 100644 index 0621a618521..00000000000 --- a/changes/4627.removal +++ /dev/null @@ -1,6 +0,0 @@ -Logic functions removed: - ``dashboard_activity_list_html`` ``organization_activity_list_html`` - ``user_activity_list_html`` ``package_activity_list_html`` - ``group_activity_list_html`` ``organization_activity_list_html`` - ``recently_changed_packages_activity_list_html`` - ``dashboard_activity_list_html`` ``activity_detail_list`` diff --git a/changes/4711.bugfix b/changes/4711.bugfix deleted file mode 100644 index 14f88353bc2..00000000000 --- a/changes/4711.bugfix +++ /dev/null @@ -1 +0,0 @@ -Emails not sent from flask routes diff --git a/changes/4770.misc b/changes/4770.misc deleted file mode 100644 index a324313ebf0..00000000000 --- a/changes/4770.misc +++ /dev/null @@ -1,2 +0,0 @@ -Give users the option to define which page they want to be redirected -to after logging in via `ckan.route_after_login` config variable. diff --git a/changes/4779.removal b/changes/4779.removal deleted file mode 100644 index ea7607d7f27..00000000000 --- a/changes/4779.removal +++ /dev/null @@ -1 +0,0 @@ -Remove Bootstrap 2 templates diff --git a/changes/4781.migration b/changes/4781.migration deleted file mode 100644 index 1849034fb69..00000000000 --- a/changes/4781.migration +++ /dev/null @@ -1 +0,0 @@ -When ``ckan.cache_enabled`` is set to ``False`` (default) all requests include the ``Cache-control: private`` header. If ``ckan.cache_enabled`` is set to ``True``, when the user is not logged in and there is no session data, a ``Cache-Control: public`` header will be added. For all other requests the ``Cache-control: private`` header will be added. Note that you will also need to set the ``ckan.cache_expires`` config option to allow caching of requests. diff --git a/changes/4781.misc b/changes/4781.misc deleted file mode 100644 index f7297817a9e..00000000000 --- a/changes/4781.misc +++ /dev/null @@ -1 +0,0 @@ -Add cache control headers to flask diff --git a/changes/4784.migration b/changes/4784.migration deleted file mode 100644 index 8d0ceb0d205..00000000000 --- a/changes/4784.migration +++ /dev/null @@ -1,16 +0,0 @@ -When upgrading from previous CKAN versions, the Activity Stream needs a -migrate_package_activity.py running for displaying the history of dataset -changes. This can be performed while CKAN is running or stopped (whereas the -standard `paster db upgrade` migrations need CKAN to be stopped). Ideally it -is run before CKAN is upgraded, but it can be run afterwards. If running -previous versions or this version of CKAN, download and run -migrate_package_activity.py like this:: - - cd /usr/lib/ckan/default/src/ckan/ - wget https://raw.githubusercontent.com/ckan/ckan/3484_revision_ui_removal2/ckan/migration/migrate_package_activity.py - wget https://raw.githubusercontent.com/ckan/ckan/3484_revision_ui_removal2/ckan/migration/revision_legacy_code.py - python migrate_package_activity.py -c /etc/ckan/production.ini - -Future versions of CKAN are likely to need a slightly different procedure. -Full info about this migration is found here: -https://github.com/ckan/ckan/wiki/Migrate-package-activity diff --git a/changes/4796.migration b/changes/4796.migration deleted file mode 100644 index ab97954b791..00000000000 --- a/changes/4796.migration +++ /dev/null @@ -1,19 +0,0 @@ -This version requires changes to the ``who.ini`` configuration file. If your -setup doesn't use the one bundled with this repo, you will have to manually -change the following lines:: - - use = ckan.lib.auth_tkt:make_plugin - -to:: - - use = ckan.lib.repoze_plugins.auth_tkt:make_plugin - -And also:: - - use = repoze.who.plugins.friendlyform:FriendlyFormPlugin - -to:: - - use = ckan.lib.repoze_plugins.friendly_form:FriendlyFormPlugin - -Otherwise, if you are using symbolinc link to ``who.ini`` under vcs, no changes required. diff --git a/changes/4821.bugfix b/changes/4821.bugfix deleted file mode 100644 index 636dc4d0b7a..00000000000 --- a/changes/4821.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Admin of organization can add himself as a member/editor to the -organization and lose admin rights diff --git a/changes/4826.bugfix b/changes/4826.bugfix deleted file mode 100644 index ebdcb40d6e9..00000000000 --- a/changes/4826.bugfix +++ /dev/null @@ -1 +0,0 @@ -Error when posting empty array with type json using datastore_create diff --git a/changes/4831.bugfix b/changes/4831.bugfix deleted file mode 100644 index e41b9f2f816..00000000000 --- a/changes/4831.bugfix +++ /dev/null @@ -1 +0,0 @@ -ValueError when you configure exception emails diff --git a/changes/4936.misc b/changes/4936.misc deleted file mode 100644 index 843d1a9b2f5..00000000000 --- a/changes/4936.misc +++ /dev/null @@ -1 +0,0 @@ -Create recline_view on ods files by default diff --git a/changes/4987.bugfix b/changes/4987.bugfix deleted file mode 100644 index 8660358c6fb..00000000000 --- a/changes/4987.bugfix +++ /dev/null @@ -1 +0,0 @@ -Dataset counts incorrect on Groups listing diff --git a/changes/4996.misc b/changes/4996.misc deleted file mode 100644 index b896e052bb7..00000000000 --- a/changes/4996.misc +++ /dev/null @@ -1 +0,0 @@ -Replase nosetests with pytest diff --git a/changes/5012.misc b/changes/5012.misc deleted file mode 100644 index 65459669ace..00000000000 --- a/changes/5012.misc +++ /dev/null @@ -1 +0,0 @@ -Make creating new tags in autocomplete module optional diff --git a/changes/5024.misc b/changes/5024.misc deleted file mode 100644 index de2d5846169..00000000000 --- a/changes/5024.misc +++ /dev/null @@ -1 +0,0 @@ -Allow reply to emails diff --git a/changes/5034.misc b/changes/5034.misc deleted file mode 100644 index ed40ef8be71..00000000000 --- a/changes/5034.misc +++ /dev/null @@ -1 +0,0 @@ -Improve and reorder resource_formats.json diff --git a/changes/5096.feature b/changes/5096.feature deleted file mode 100644 index 487c6872096..00000000000 --- a/changes/5096.feature +++ /dev/null @@ -1 +0,0 @@ -Added Python 3 support diff --git a/changes/5100.misc b/changes/5100.misc deleted file mode 100644 index 362a6afb2f9..00000000000 --- a/changes/5100.misc +++ /dev/null @@ -1 +0,0 @@ -Email unique validator diff --git a/changes/5103.misc b/changes/5103.misc deleted file mode 100644 index 20733047b83..00000000000 --- a/changes/5103.misc +++ /dev/null @@ -1 +0,0 @@ -Preview for multimedia files diff --git a/changes/5112.misc b/changes/5112.misc deleted file mode 100644 index d1a50062174..00000000000 --- a/changes/5112.misc +++ /dev/null @@ -1 +0,0 @@ -Allow extensions to define Click commands diff --git a/changes/5112.removal b/changes/5112.removal deleted file mode 100644 index 2784890f644..00000000000 --- a/changes/5112.removal +++ /dev/null @@ -1,3 +0,0 @@ -Extensions that add CLI commands should note the deprecation of -``ckan.lib.cli.CkanCommand`` and all other helpers in ckan.lib.cli. -Extensions should instead implement CLIs using the new IClick interface. diff --git a/changes/5127.misc b/changes/5127.misc deleted file mode 100644 index 637d32e3e63..00000000000 --- a/changes/5127.misc +++ /dev/null @@ -1 +0,0 @@ -Add organization and group purge diff --git a/changes/5132.misc b/changes/5132.misc deleted file mode 100644 index bb83ad7e787..00000000000 --- a/changes/5132.misc +++ /dev/null @@ -1 +0,0 @@ -HTML emails diff --git a/changes/5146.feature b/changes/5146.feature deleted file mode 100644 index e368321bb3e..00000000000 --- a/changes/5146.feature +++ /dev/null @@ -1,8 +0,0 @@ -API Tokens: an alternative to API keys. Tokens can be created and -removed on demand(check :ref:`api authentication>`) and there is no -restriction on the maximum number of tokens per user. Consider using -tokens instead of API keys and create a separate token for each -use-case instead of sharing the same token between multiple -clients. By default API Tokens are JWT, but the goal is to make them -as customizable as possible, so alternative formats can be implemented -using `ckan.plugins.interfaces.IApiToken` interface. diff --git a/changes/5147.bugfix b/changes/5147.bugfix deleted file mode 100644 index 8ce7ca4bc2e..00000000000 --- a/changes/5147.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix broken layout in organization bulk_process diff --git a/changes/5150.misc b/changes/5150.misc deleted file mode 100644 index 0f8cfcb3334..00000000000 --- a/changes/5150.misc +++ /dev/null @@ -1 +0,0 @@ -Unified workflow for creating/applying DB migrations from extensions diff --git a/changes/5169.feature b/changes/5169.feature new file mode 100644 index 00000000000..fc39b2358b4 --- /dev/null +++ b/changes/5169.feature @@ -0,0 +1,2 @@ +Add blanket implementations - decorators providing common +implementation of simple interfaces. diff --git a/changes/5172.bugfix b/changes/5172.bugfix deleted file mode 100644 index 3b2da76e739..00000000000 --- a/changes/5172.bugfix +++ /dev/null @@ -1 +0,0 @@ -Index template with template path instead of numeric index diff --git a/changes/5189.misc b/changes/5189.misc deleted file mode 100644 index 7ca066dc098..00000000000 --- a/changes/5189.misc +++ /dev/null @@ -1 +0,0 @@ -Use current package_type for urls diff --git a/changes/5195.misc b/changes/5195.misc deleted file mode 100644 index 34b87e7a1f5..00000000000 --- a/changes/5195.misc +++ /dev/null @@ -1 +0,0 @@ -Werkzeug dev server improvements diff --git a/changes/5208.misc b/changes/5208.misc deleted file mode 100644 index aca5bfaff24..00000000000 --- a/changes/5208.misc +++ /dev/null @@ -1 +0,0 @@ -Allow passing arguments to the RQ enqueue_call function diff --git a/changes/5223.misc b/changes/5223.misc deleted file mode 100644 index 5d5f6af6e39..00000000000 --- a/changes/5223.misc +++ /dev/null @@ -1 +0,0 @@ -Add option to configure labels of next/prev page button and pager format. diff --git a/changes/5236.bugfix b/changes/5236.bugfix deleted file mode 100644 index 39069a355f4..00000000000 --- a/changes/5236.bugfix +++ /dev/null @@ -1 +0,0 @@ -Add metadata_modified field to resource diff --git a/changes/5264.removal b/changes/5264.removal deleted file mode 100644 index 7b63095a40f..00000000000 --- a/changes/5264.removal +++ /dev/null @@ -1 +0,0 @@ -Remove paster CLI diff --git a/changes/5272.feature b/changes/5272.feature deleted file mode 100644 index bfd4e4bbcab..00000000000 --- a/changes/5272.feature +++ /dev/null @@ -1 +0,0 @@ -Users can now upload or link to custom profile pictures. By default, if a user picture is not provided it will fall back to gravatar. Alternatively, gravatar can be completely disabled by setting ``ckan.gravatar_default = disabled``. In that case a placeholder image is shown instead, which can be customized by overriding the ``templates/user/snippets/placeholder.html`` template. diff --git a/changes/5281.bugfix b/changes/5281.bugfix deleted file mode 100644 index 74280569317..00000000000 --- a/changes/5281.bugfix +++ /dev/null @@ -1 +0,0 @@ -Send the right URL of CKAN to datapusher diff --git a/changes/5303.misc b/changes/5303.misc deleted file mode 100644 index b66bb91e9aa..00000000000 --- a/changes/5303.misc +++ /dev/null @@ -1 +0,0 @@ -DevServer: threaded mode and extra files diff --git a/changes/5314.misc b/changes/5314.misc deleted file mode 100644 index deb20ed870d..00000000000 --- a/changes/5314.misc +++ /dev/null @@ -1 +0,0 @@ -Make default sorting configurable diff --git a/changes/5339.bugfix b/changes/5339.bugfix deleted file mode 100644 index 828a1cd5248..00000000000 --- a/changes/5339.bugfix +++ /dev/null @@ -1 +0,0 @@ -Multiline translation strings not translated diff --git a/changes/5345.misc b/changes/5345.misc deleted file mode 100644 index 89b1566e38e..00000000000 --- a/changes/5345.misc +++ /dev/null @@ -1 +0,0 @@ -Allow initial values in group form diff --git a/changes/5346.feature b/changes/5346.feature deleted file mode 100644 index bad8c3633b7..00000000000 --- a/changes/5346.feature +++ /dev/null @@ -1,7 +0,0 @@ -Dataset collaborators: In addition to traditional organization-based permissions, CKAN instances can also enable -the dataset collaborators feature, which allows dataset-level authorization. This provides -more granular control over who can access and modify datasets that belong to an organization, -or allows authorization setups not based on organizations. It works by allowing users with -appropriate permissions to give permissions to other users over individual datasets, regardless -of what organization they belong to. To learn more about how to enable it and the different -configuration options available, check the documentation on :ref:`dataset_collaborators`. diff --git a/changes/5359.feature b/changes/5359.feature new file mode 100644 index 00000000000..4dcb5023f51 --- /dev/null +++ b/changes/5359.feature @@ -0,0 +1 @@ +Add signals support diff --git a/changes/5360.misc b/changes/5360.misc deleted file mode 100644 index 6571ccb9e5b..00000000000 --- a/changes/5360.misc +++ /dev/null @@ -1 +0,0 @@ -Make ckan more accessible diff --git a/changes/5373.bugfix b/changes/5373.bugfix deleted file mode 100644 index 04d612dc0c8..00000000000 --- a/changes/5373.bugfix +++ /dev/null @@ -1 +0,0 @@ -Allow repeaded params in h.add_url_param diff --git a/changes/5376.misc b/changes/5376.misc deleted file mode 100644 index 12608c65f30..00000000000 --- a/changes/5376.misc +++ /dev/null @@ -1 +0,0 @@ -Update date formatters diff --git a/changes/5382.feature b/changes/5382.feature deleted file mode 100644 index 4f34f43ca1d..00000000000 --- a/changes/5382.feature +++ /dev/null @@ -1 +0,0 @@ -Add `plugin_extras` field allowing extending User object for internal use diff --git a/changes/5398.misc b/changes/5398.misc deleted file mode 100644 index 9c0b0d7e746..00000000000 --- a/changes/5398.misc +++ /dev/null @@ -1 +0,0 @@ -Allow multiple `ext_*` params in search views diff --git a/changes/5417.bugfix b/changes/5417.bugfix deleted file mode 100644 index 54224ec9e1e..00000000000 --- a/changes/5417.bugfix +++ /dev/null @@ -1 +0,0 @@ -Accept timestamps with seconds having less than 6 decimals diff --git a/changes/5420.bugfix b/changes/5420.bugfix deleted file mode 100644 index adc21015558..00000000000 --- a/changes/5420.bugfix +++ /dev/null @@ -1 +0,0 @@ -RTL css fixes diff --git a/changes/5432.bugfix b/changes/5432.bugfix deleted file mode 100644 index d06c920b71a..00000000000 --- a/changes/5432.bugfix +++ /dev/null @@ -1 +0,0 @@ -Prevent account presence exposure when `ckan.auth.public_user_details = false` diff --git a/changes/5436.bugfix b/changes/5436.bugfix deleted file mode 100644 index 8ee3c63e1b7..00000000000 --- a/changes/5436.bugfix +++ /dev/null @@ -1 +0,0 @@ -`ckan.i18n_directory` config option ignored in Flask app. diff --git a/changes/5453.bugfix b/changes/5453.bugfix deleted file mode 100644 index 0b4925b2571..00000000000 --- a/changes/5453.bugfix +++ /dev/null @@ -1 +0,0 @@ -Allow lists in resource extras diff --git a/changes/5458.migration b/changes/5458.migration deleted file mode 100644 index 672c40d87b0..00000000000 --- a/changes/5458.migration +++ /dev/null @@ -1 +0,0 @@ -The minimum PostgreSQL version required starting from this version is 9.5 diff --git a/changes/5464.misc b/changes/5464.misc deleted file mode 100644 index c29365c2ad5..00000000000 --- a/changes/5464.misc +++ /dev/null @@ -1 +0,0 @@ -Always 404 on non-existing user lookup diff --git a/changes/5490.feature b/changes/5490.feature new file mode 100644 index 00000000000..c2c9212cff5 --- /dev/null +++ b/changes/5490.feature @@ -0,0 +1 @@ +Add functionality for the user_show to fetch own details when logged in without passing id \ No newline at end of file diff --git a/changes/5727.bugfix b/changes/5727.bugfix new file mode 100644 index 00000000000..56fb6f6be36 --- /dev/null +++ b/changes/5727.bugfix @@ -0,0 +1 @@ +Fix the datapusher trigger in case of resource_update via API \ No newline at end of file diff --git a/changes/5831.feature b/changes/5831.feature new file mode 100644 index 00000000000..bd4ccbb9381 --- /dev/null +++ b/changes/5831.feature @@ -0,0 +1,4 @@ +``datastore_info`` now returns more detailed info. It returns database-level metadata in addition +to rowcount (aliases, id, size, index_size, db_size and table_type), and the data dictionary with +database-level schemata (native_type, index_name, is_index, notnull & uniquekey). +See the documentation at :py:func:`~ckanext.datastore.logic.action.datastore_info` diff --git a/changes/5832.feature b/changes/5832.feature new file mode 100644 index 00000000000..843b60de3ab --- /dev/null +++ b/changes/5832.feature @@ -0,0 +1 @@ +``datastore_info`` now works with aliases, and can be used to dereference aliases. diff --git a/changes/5868.feature b/changes/5868.feature new file mode 100644 index 00000000000..9de0fa76c6c --- /dev/null +++ b/changes/5868.feature @@ -0,0 +1 @@ +Add CLI commands for API Token management diff --git a/changes/5888.bugfix b/changes/5888.bugfix new file mode 100644 index 00000000000..2c762110aae --- /dev/null +++ b/changes/5888.bugfix @@ -0,0 +1 @@ +package_revise now returns some errors in normal keys instead of under 'message' diff --git a/changes/5924.feature b/changes/5924.feature new file mode 100644 index 00000000000..31204dc7e89 --- /dev/null +++ b/changes/5924.feature @@ -0,0 +1 @@ +CKAN is typed now diff --git a/changes/6000.bugfix b/changes/6000.bugfix new file mode 100644 index 00000000000..f109a599eb5 --- /dev/null +++ b/changes/6000.bugfix @@ -0,0 +1 @@ +Allow multi-level config inheritance diff --git a/changes/6008.bugfix b/changes/6008.bugfix new file mode 100644 index 00000000000..a19abf18690 --- /dev/null +++ b/changes/6008.bugfix @@ -0,0 +1 @@ +Fix Chinese locales. Note that the URLs for the `zh_CN` and `zh_TW` locales have changed but there are redirects in place, eg http://localhost:5000/zh_CN/dataset -> http://localhost:5000/zh_Hans_CN/dataset diff --git a/changes/6028.bugfix b/changes/6028.bugfix new file mode 100644 index 00000000000..2bbd9faf175 --- /dev/null +++ b/changes/6028.bugfix @@ -0,0 +1 @@ +Fix performance bottleneck in activity queries diff --git a/changes/6048.removal b/changes/6048.removal new file mode 100644 index 00000000000..ffc3c019450 --- /dev/null +++ b/changes/6048.removal @@ -0,0 +1,2 @@ +Only user-defined functions can be used as validators. Attempt to use +mock-object, built-in function or class will cause TypeError. diff --git a/changes/6084.bugfix b/changes/6084.bugfix new file mode 100644 index 00000000000..4cda01ba879 --- /dev/null +++ b/changes/6084.bugfix @@ -0,0 +1 @@ +Keep repeatable facets inside pagination links diff --git a/changes/6120.bugfix b/changes/6120.bugfix new file mode 100644 index 00000000000..967ef0da43f --- /dev/null +++ b/changes/6120.bugfix @@ -0,0 +1 @@ +Consistent CLI behavior when when no command provided and when using `--help` options diff --git a/changes/6175.feature b/changes/6175.feature new file mode 100644 index 00000000000..b55034e8598 --- /dev/null +++ b/changes/6175.feature @@ -0,0 +1 @@ +Migrated preprocessor from LESS to SCSS for preliminary work for Bootstrap upgrade. diff --git a/changes/6192.bugfix b/changes/6192.bugfix new file mode 100644 index 00000000000..8315bd542bb --- /dev/null +++ b/changes/6192.bugfix @@ -0,0 +1,15 @@ +Variables from extended config files (``use = config:...``) has lower precedence. +In the following example:: + + ;; a.ini + output = %(var)s + + ;; b.ini + use = config:a.ini + var = B + + ;; c.ini + use = config:b.ini + var = C + +final value of the ``output`` config option will be ``C``. diff --git a/changes/6192.misc b/changes/6192.misc new file mode 100644 index 00000000000..de394cda448 --- /dev/null +++ b/changes/6192.misc @@ -0,0 +1 @@ +Environment variables prefixed with `CKAN_` can be used as variables inside config file via `option = %(CKAN_***)s` diff --git a/changes/6226.feature b/changes/6226.feature new file mode 100644 index 00000000000..3a8d7e2443a --- /dev/null +++ b/changes/6226.feature @@ -0,0 +1 @@ +Add extensible snippet for resource uploads diff --git a/changes/6247.removal b/changes/6247.removal new file mode 100644 index 00000000000..a824957c48f --- /dev/null +++ b/changes/6247.removal @@ -0,0 +1,3 @@ +Legacy API keys are no longer supported for Authentication and have been removed +from the UI. API Tokens should be used instead. See :ref:`api authentication` for +more details diff --git a/changes/6263.removal b/changes/6263.removal new file mode 100644 index 00000000000..dd0629342a5 --- /dev/null +++ b/changes/6263.removal @@ -0,0 +1,2 @@ +`build_nav_main()`, `build_nav_icon()` and `build_nav()` helpers no longer support +Pylons route syntax. eg use `dataset.search` instead of `controller=dataset, action=search`. diff --git a/changes/6272.removal b/changes/6272.removal new file mode 100644 index 00000000000..f8a842fbc96 --- /dev/null +++ b/changes/6272.removal @@ -0,0 +1,3 @@ +The following old helper functions have been removed and are no longer available: +``submit()``, ``radio()``, ``icon_url()``, ``icon_html()``, ``icon()``, ``resource_icon()``, +``format_icon()``, ``button_attr()``, ``activity_div()`` diff --git a/changes/6287.misc b/changes/6287.misc new file mode 100644 index 00000000000..4ec4363ff36 --- /dev/null +++ b/changes/6287.misc @@ -0,0 +1 @@ +CLI command less is now renamed to sass as the preprocessor was changed in #6175. diff --git a/changes/6307.feature b/changes/6307.feature new file mode 100644 index 00000000000..f3a66f267f1 --- /dev/null +++ b/changes/6307.feature @@ -0,0 +1,5 @@ +Migrated bootstrap v3 to v5 for the default CKAN theme. Bootstrap v3 templates are still available +for use by specifying the base template folder in the configuration. + +`ckan.base_public_folder=public-bs3` +`ckan.base_templates_folder=templates-bs3` diff --git a/changes/6329.bugfix b/changes/6329.bugfix new file mode 100644 index 00000000000..fd89ad84354 --- /dev/null +++ b/changes/6329.bugfix @@ -0,0 +1,2 @@ +Restore error traceback for `search-index rebuild -i` CLI command + diff --git a/changes/6335.feature b/changes/6335.feature new file mode 100644 index 00000000000..624e541b3e9 --- /dev/null +++ b/changes/6335.feature @@ -0,0 +1 @@ +Test factories extends SQLAlchemy factory, are available via fixtures and produce more random entities using faker library. diff --git a/changes/6340.bugfix b/changes/6340.bugfix new file mode 100644 index 00000000000..edd8b857a4d --- /dev/null +++ b/changes/6340.bugfix @@ -0,0 +1,2 @@ +Prevent Traceback to logged for HTTP Exception until debug is true +Add the HTTP status Code in logging for HTTP requests \ No newline at end of file diff --git a/changes/6356.bugfix b/changes/6356.bugfix new file mode 100644 index 00000000000..49c7e82026c --- /dev/null +++ b/changes/6356.bugfix @@ -0,0 +1,2 @@ + +Improve rendering data types in resource view diff --git a/changes/6406.bugfix b/changes/6406.bugfix new file mode 100644 index 00000000000..796cc599e16 --- /dev/null +++ b/changes/6406.bugfix @@ -0,0 +1 @@ +Snippet names rendered into HTML as comments in non-debug mode. diff --git a/changes/6414.bugfix b/changes/6414.bugfix new file mode 100644 index 00000000000..0a86a85f709 --- /dev/null +++ b/changes/6414.bugfix @@ -0,0 +1 @@ +h.remove_url_param fail with minimal set of params diff --git a/changes/6466.feature b/changes/6466.feature new file mode 100644 index 00000000000..b4381c7a9c6 --- /dev/null +++ b/changes/6466.feature @@ -0,0 +1 @@ +CKAN now records the last time a user was active on the site. The minimum interval between records can be controlled with the :ref:`ckan.user.last_active_interval` config option. diff --git a/changes/6467.feature b/changes/6467.feature new file mode 100644 index 00000000000..669eceab772 --- /dev/null +++ b/changes/6467.feature @@ -0,0 +1,56 @@ +:ref:`declare-config-options`: declare configuration options to ensure validation and default values. + +All the CKAN configuration options are validated and converted to the +expected type during the application startup. That's how the behavior has changed:: + + debug = config.get("debug") + + # CKAN <= v2.9 + assert type(debug) is str + assert debug == "false" # or any value that is specified in the config file + + # CKAN >= v2.10 + assert type(debug) is bool + assert debug is False # or ``True`` + +The ``aslist``, ``asbool``, ``asint`` converters from ``ckan.plugins.toolkit`` will keep the current behaviour:: + + # produces the same result in v2.9 and v2.10 + assert tk.asbool(config.get("debug")) is False + assert tk.asint(config.get("ckan.devserver.port")) == 5000 + assert tk.aslist(config.get("ckan.plugins")) == ["stats"] + +If you are using custom logic, the code requires a review. For example, the +following code will produce an ``AttributeError`` exception, because ``ckan.plugins`` is +converted into a list during the application's startup:: + + # AttributeError + plugins = config.get("ckan.plugins").split() + +Depending on the desired backward compatibility, one of the following expressions +can be used instead:: + + # if both v2.9 and v2.10 are supported + plugins = tk.aslist(config.get("ckan.plugins")) + + # if only v2.10 is supported + plugins = config.get("ckan.plugins") + +The second major change affects default values for configuration options. Starting from CKAN 2.10, +the majority of the config options have a declared default value. It means that +whenever you invoke ``config.get`` method, the *declared default* value is +returned instead of ``None``. Example:: + + # CKAN v2.9 + assert config.get("search.facets.limit") is None + + # CKAN v2.10 + assert config.get("search.facets.limit") == 10 + +The second argument to ``config.get`` should be only used to get +the value of a missing *undeclared* option:: + + assert config.get("not.declared.and.missing.from.config", 1) == 1 + +The above is the same for any extension that *declares* its config options +using ``IConfigDeclaration`` interface or ``config_declarations`` blanket. diff --git a/changes/6477.bugfix b/changes/6477.bugfix new file mode 100644 index 00000000000..347393e68ea --- /dev/null +++ b/changes/6477.bugfix @@ -0,0 +1,3 @@ +Type of uploads for group and user image can be restricted via the +`ckan.upload.{object_type}.types` and `ckan.upload.{object_type}.mimetypes` +config options (eg `ckan.upload.group.types`, `ckan.upload.user.mimetypes`) diff --git a/changes/6501.removal b/changes/6501.removal new file mode 100644 index 00000000000..03ef75c30a0 --- /dev/null +++ b/changes/6501.removal @@ -0,0 +1,25 @@ + +The following methods are deprecated and should be replaced with their +respective new versions in the plugin interfaces: + +- `ckan.plugins.interfaces.IResourceController`: + + - change `before_create` to `before_resource_create` + - change `after_create` to `after_resource_create` + - change `before_update` to `before_resource_update` + - change `after_update` to `after_resource_update` + - change `before_delete` to `before_resource_delete` + - change `after_delete` to `after_resource_delete` + - change `before_show` to `before_resource_show` + +- `ckan.plugins.interfaces.IPackageController`: + + - change `after_create` to `after_dataset_create` + - change `after_update` to `after_dataset_update` + - change `after_delete` to `after_dataset_delete` + - change `after_show` to `after_dataset_show` + - change `before_search` to `before_dataset_search` + - change `after_search` to `after_dataset_search` + - change `before_index` to `before_dataset_index` + +| diff --git a/changes/6504.removal b/changes/6504.removal new file mode 100644 index 00000000000..4f4f98c651e --- /dev/null +++ b/changes/6504.removal @@ -0,0 +1,3 @@ +The ``ckan seed`` command has been removed in favour of ``ckan generate fake-data`` +for generating test entities in the database. Refer to ``ckan generate fake-data --help`` +for some usage examples. diff --git a/changes/6507.changes b/changes/6507.changes new file mode 100644 index 00000000000..9a22123ac8f --- /dev/null +++ b/changes/6507.changes @@ -0,0 +1,4 @@ +`group_extra`, `package`, `package_extra`, `resource` and `tag` entities no longer have +a default `order_by` in the query results as it has been deprecated in SQLAlchemy. +If you require a specific order (eg by package name), you must explicitly +declare it in the desired query. \ No newline at end of file diff --git a/changes/6519.bugfix b/changes/6519.bugfix new file mode 100644 index 00000000000..42e880ae20e --- /dev/null +++ b/changes/6519.bugfix @@ -0,0 +1,2 @@ +``*_patch`` actions call their ``*_update`` equivalents via ``get_action`` +allowing plugins to override them consistently diff --git a/changes/6535.misc b/changes/6535.misc new file mode 100644 index 00000000000..bfa974b4eed --- /dev/null +++ b/changes/6535.misc @@ -0,0 +1 @@ +Support including file attachments when sending emails diff --git a/changes/6557.misc b/changes/6557.misc new file mode 100644 index 00000000000..baead92050a --- /dev/null +++ b/changes/6557.misc @@ -0,0 +1,2 @@ +Activities now receive the full dict of the object they refer to in their ``data`` section. This allows greater flexibility when creating custom activities from plugins. + diff --git a/changes/6594.removal b/changes/6594.removal new file mode 100644 index 00000000000..46fce059356 --- /dev/null +++ b/changes/6594.removal @@ -0,0 +1,2 @@ +The `IRoutes` interface has been removed since it was part of the old Pylons +architecture. \ No newline at end of file diff --git a/changes/6628.removal b/changes/6628.removal new file mode 100644 index 00000000000..cf2d68b5f20 --- /dev/null +++ b/changes/6628.removal @@ -0,0 +1 @@ +Remove ckan.cache_validated_datasets config \ No newline at end of file diff --git a/changes/6637.bugfix b/changes/6637.bugfix new file mode 100644 index 00000000000..61318d9a40c --- /dev/null +++ b/changes/6637.bugfix @@ -0,0 +1 @@ +Fixed and simplified organization and group forms breadcrumb inheritance diff --git a/changes/6639.removal b/changes/6639.removal new file mode 100644 index 00000000000..34a0f30b486 --- /dev/null +++ b/changes/6639.removal @@ -0,0 +1 @@ +Remove ckan.search.automatic_indexing config \ No newline at end of file diff --git a/changes/6641.feature b/changes/6641.feature new file mode 100644 index 00000000000..c6abeb2fd9b --- /dev/null +++ b/changes/6641.feature @@ -0,0 +1 @@ +Make private dataset behaviour configurable diff --git a/changes/6648.removal b/changes/6648.removal new file mode 100644 index 00000000000..23176b57d12 --- /dev/null +++ b/changes/6648.removal @@ -0,0 +1,2 @@ +The `PluginMapperExtension` has been removed since it was no longer used in core +and it had a deprecated dependency. \ No newline at end of file diff --git a/changes/6687.removal b/changes/6687.removal new file mode 100644 index 00000000000..fe8061d44cd --- /dev/null +++ b/changes/6687.removal @@ -0,0 +1 @@ +Remove deprecated `fields` parameter in `resource_search` method. \ No newline at end of file diff --git a/changes/6698.bugfix b/changes/6698.bugfix new file mode 100644 index 00000000000..01a9d07cebc --- /dev/null +++ b/changes/6698.bugfix @@ -0,0 +1 @@ +Ensure that locale exists on i18n JS API diff --git a/changes/6699.removal b/changes/6699.removal new file mode 100644 index 00000000000..8add4bff794 --- /dev/null +++ b/changes/6699.removal @@ -0,0 +1,2 @@ +The ISession interface has been removed from CKAN. To extend SQLAlchemy use +event listeners instead. diff --git a/changes/6746.migration b/changes/6746.migration new file mode 100644 index 00000000000..a0cabe48ae5 --- /dev/null +++ b/changes/6746.migration @@ -0,0 +1 @@ +The language code for the Norwegian language has been updated from ``no`` to ``nb_NO``. There are redirects in place from the old code to the new one for localized URLs, but please update your links. If you were using the old ``no`` code in a config option like ``ckan.default_locale`` or ``ckan.locales_offered`` you will need to update the value to ``nb_NO``. diff --git a/changes/6747.misc b/changes/6747.misc new file mode 100644 index 00000000000..42a68aed39e --- /dev/null +++ b/changes/6747.misc @@ -0,0 +1,7 @@ +Reworked the JavaScript for the view filters to allow for special characters as well as colons and pipes, which previously caused errors. Added a new helper to easily decode the new flattened filter string. + +- New helper `decode_view_request_filters` + - Returns a `dictionary` of the decoded filter names and their values in a `list`. +- View Filters JS related code will now encode the filter names and values before flattening into a colon (`:`) and piped (`|`) string. + - Allowing for colons and pipes to be in the filter names and values. + - Allowing special characters to be in the filter names and values (such as accented chars). \ No newline at end of file diff --git a/changes/6748.changes b/changes/6748.changes new file mode 100644 index 00000000000..7485154557f --- /dev/null +++ b/changes/6748.changes @@ -0,0 +1 @@ +`User.by_email` method no longer returns a list, instead, it returns a single user. \ No newline at end of file diff --git a/changes/6765.removal b/changes/6765.removal new file mode 100644 index 00000000000..a45862a09f5 --- /dev/null +++ b/changes/6765.removal @@ -0,0 +1,2 @@ +`unselected_facet_items` helper has been removed. You can use +`get_facet_items_dict` with exclude_active=True instead. diff --git a/changes/6790.misc b/changes/6790.misc new file mode 100644 index 00000000000..0e0f860a22c --- /dev/null +++ b/changes/6790.misc @@ -0,0 +1,15 @@ +Activites extracted into `activity` plugin. As result, activity tabs from the +dataset, group, organization and user sections are not shown by default; new +activities are not recorded; all the API actions related to activity are not +available. + +In order to restore all the mentioned features, add `activity` plugin to the +`ckan.plugins` config option. + +This change doesn't affect data stored in the DB. Even if `activity` plugin is +not enabled, old activities are not removed and will be available at any moment +in the future, after enabling the `activity` plugin. + +Import changes: + +* `ckan.model.Activity` -> `ckanext.activity.model.Activity` diff --git a/changes/6793.feature b/changes/6793.feature new file mode 100644 index 00000000000..3f35ab3da0e --- /dev/null +++ b/changes/6793.feature @@ -0,0 +1 @@ +allow _id for datastore_upsert unique key diff --git a/changes/6817.bugfix b/changes/6817.bugfix new file mode 100644 index 00000000000..8b7ac89be3b --- /dev/null +++ b/changes/6817.bugfix @@ -0,0 +1,7 @@ +Fix theme settings. Options which were used to specify a CSS file +with a base theme are replaced. Use altenatives below in order +to specify **asset** with a base theme for application: + +* `ckan.main_css` replaced by `ckan.theme` +* `ckan.i18n.rtl_css` replaced by `ckan.i18n.rtl_theme` + diff --git a/changes/6897.changes b/changes/6897.changes new file mode 100644 index 00000000000..a06091033de --- /dev/null +++ b/changes/6897.changes @@ -0,0 +1,3 @@ +Removed the password creation logic for user, when inviting a user. +Added new schema `create_user_for_user_invite` that disables the password +field on user creation. \ No newline at end of file diff --git a/changes/6898.changes b/changes/6898.changes new file mode 100644 index 00000000000..5e46448fdba --- /dev/null +++ b/changes/6898.changes @@ -0,0 +1 @@ +Reverse the order in which IConfigurer plugins are read to be consistent with other hooks. \ No newline at end of file diff --git a/changes/6919.feature b/changes/6919.feature new file mode 100644 index 00000000000..50d21a54bcd --- /dev/null +++ b/changes/6919.feature @@ -0,0 +1 @@ +Added new command `ckan shell` that opens an interactive python shell with the Flask's application context preloaded (among other useful objects). \ No newline at end of file diff --git a/changes/6920.feature b/changes/6920.feature new file mode 100644 index 00000000000..4227169f1e6 --- /dev/null +++ b/changes/6920.feature @@ -0,0 +1,12 @@ +Added CSRF protection that would protect all the forms against Cross-Site Request Forgery attacks. +This feature is enabled by default in CKAN core, extensions are excluded from CSRF protection till they are ready to implement the csrf_token to their forms. + +To enable the CSRF protection in your extensions you would need to set: + +`ckan.csrf_protection.ignore_extensions=False` + +and to set csrf_token in your forms: + +`{{ h.csrf_input() }}` + +See the documentation at `https://docs.ckan.org/en/latest/extensions/best-practices.html` for more info. diff --git a/changes/6956.misc b/changes/6956.misc new file mode 100644 index 00000000000..049a47fb39c --- /dev/null +++ b/changes/6956.misc @@ -0,0 +1 @@ +Non-sysadmin users are no longer able to change their own state diff --git a/changes/6961.misc b/changes/6961.misc new file mode 100644 index 00000000000..9ebef4bd55d --- /dev/null +++ b/changes/6961.misc @@ -0,0 +1 @@ + The "rank" field is no longer returned in datastore_search results unless explicitly defined in the fields parameter diff --git a/changes/6991.changes b/changes/6991.changes new file mode 100644 index 00000000000..9e219e52bb1 --- /dev/null +++ b/changes/6991.changes @@ -0,0 +1,2 @@ +anonymous users are no longer redirected to the login page when 403/401 happens, instead, +we are showing the error page with a link to the login page. diff --git a/changes/7003.misc b/changes/7003.misc new file mode 100644 index 00000000000..5c23f393d84 --- /dev/null +++ b/changes/7003.misc @@ -0,0 +1 @@ +Github Actions now run checks that changelog fragments are present in Pull Requests. diff --git a/changes/7011.feature b/changes/7011.feature new file mode 100644 index 00000000000..858dd309895 --- /dev/null +++ b/changes/7011.feature @@ -0,0 +1 @@ +Add `ckan.plugins.core.plugin_loaded` to the core helpers as `plugin_loaded` diff --git a/changes/7031.bugfix b/changes/7031.bugfix new file mode 100644 index 00000000000..b884f86b0f8 --- /dev/null +++ b/changes/7031.bugfix @@ -0,0 +1 @@ +prepare_dataset_blueprint: support dataset type diff --git a/changes/7039.bugfix b/changes/7039.bugfix new file mode 100644 index 00000000000..0f9a56352d9 --- /dev/null +++ b/changes/7039.bugfix @@ -0,0 +1 @@ +Changed default sort key for group and user lists from ASCII Alphebitized to new `strxfrm` helper, resulting in human-readable alphebitization. diff --git a/changes/7044.feature b/changes/7044.feature new file mode 100644 index 00000000000..cd92b7e18f0 --- /dev/null +++ b/changes/7044.feature @@ -0,0 +1,5 @@ +Added `list-orphans` and `clear-orphans` sub-commands to the `search-index` command. + +`list-orphans` will list all public package IDs which exist in the solr index, but do not exist in the database. + +`clear-orphans` will clear the search index for all the public orphaned packages. diff --git a/changes/7064.misc b/changes/7064.misc new file mode 100644 index 00000000000..bf7d895aa06 --- /dev/null +++ b/changes/7064.misc @@ -0,0 +1 @@ +Upgrade requirements to the latest version whenever possible diff --git a/changes/7066.misc b/changes/7066.misc new file mode 100644 index 00000000000..c6aa9bd76d4 --- /dev/null +++ b/changes/7066.misc @@ -0,0 +1 @@ +gzip .test_durations diff --git a/changes/7075.bugfix b/changes/7075.bugfix new file mode 100644 index 00000000000..d4f47157aa9 --- /dev/null +++ b/changes/7075.bugfix @@ -0,0 +1 @@ +Fix resource file size not updating with resource_patch diff --git a/changes/7078.removal b/changes/7078.removal new file mode 100644 index 00000000000..a1c90357cf8 --- /dev/null +++ b/changes/7078.removal @@ -0,0 +1 @@ +The Recline-based view plugins (`recline_view`, `recline_grid_view`, `recline_graph_view` and `recline_map_view`) are deprecated and will be removed in future versions. Check :ref:`data-viewer` for alternatives. diff --git a/changes/7082.bugfix b/changes/7082.bugfix new file mode 100644 index 00000000000..8d34f6002af --- /dev/null +++ b/changes/7082.bugfix @@ -0,0 +1 @@ +Revert Flask requirement from 2.2.2 to 2.0.3. \ No newline at end of file diff --git a/changes/7085.bugfix b/changes/7085.bugfix new file mode 100644 index 00000000000..3c7a4018072 --- /dev/null +++ b/changes/7085.bugfix @@ -0,0 +1 @@ +restore original plugin template directory order after update_config order change diff --git a/changes/7088.misc b/changes/7088.misc new file mode 100644 index 00000000000..aafcfad8d1d --- /dev/null +++ b/changes/7088.misc @@ -0,0 +1 @@ +Site maintainers can choose to completely ignore cookie based by using ``ckan.auth.enable_cookie_auth_in_api``. When set to False, all API requests must use :ref:`API Tokens `. Note that this is likely to break some existing JS modules from the frontend that perform API calls, so it should be used with caution. diff --git a/changes/7107.bugfix b/changes/7107.bugfix new file mode 100644 index 00000000000..79c2f3db433 --- /dev/null +++ b/changes/7107.bugfix @@ -0,0 +1 @@ +Fix urls containing unicode encoded in hex diff --git a/changes/7108.bugfix b/changes/7108.bugfix new file mode 100644 index 00000000000..d70311f9d04 --- /dev/null +++ b/changes/7108.bugfix @@ -0,0 +1 @@ +Fix a bug that causes CKAN to only register the first blueprint of plugins. \ No newline at end of file diff --git a/changes/7112.misc b/changes/7112.misc new file mode 100644 index 00000000000..db09a35603e --- /dev/null +++ b/changes/7112.misc @@ -0,0 +1,2 @@ +Creates a `fresh_context` function to allow cleaning the `context` dict preserving most importan values (`user`, `model`, etc) + diff --git a/changes/7119.bugfix b/changes/7119.bugfix new file mode 100644 index 00000000000..0bd4c4b39ed --- /dev/null +++ b/changes/7119.bugfix @@ -0,0 +1 @@ +remove old deleted resources on package_update so that performance is consistent over time (no longer degrading) diff --git a/changes/7127.misc b/changes/7127.misc new file mode 100644 index 00000000000..ac725550fe1 --- /dev/null +++ b/changes/7127.misc @@ -0,0 +1 @@ +Added the csrf_input.html in templates-bs3/snippets diff --git a/changes/7133.bugfix b/changes/7133.bugfix new file mode 100644 index 00000000000..6b447549811 --- /dev/null +++ b/changes/7133.bugfix @@ -0,0 +1 @@ +Beaker session config variables need to be initialised in a newly generated ckan config file diff --git a/changes/7134.feature b/changes/7134.feature new file mode 100644 index 00000000000..be193c0b0b9 --- /dev/null +++ b/changes/7134.feature @@ -0,0 +1 @@ +Add an index on column resource_id in table resource_view. \ No newline at end of file diff --git a/changes/7139.migration b/changes/7139.migration new file mode 100644 index 00000000000..38c6bdc0fa9 --- /dev/null +++ b/changes/7139.migration @@ -0,0 +1 @@ +Users of the Xloader or DataPusher need to provide a valid API Token in their configurations using the ``ckanext.xloader.api_token`` or ``ckan.datapusher.api_token`` keys respectively. diff --git a/changes/7150.bugfix b/changes/7150.bugfix new file mode 100644 index 00000000000..180fbd90521 --- /dev/null +++ b/changes/7150.bugfix @@ -0,0 +1 @@ +Fixed broken organization delete form diff --git a/changes/7153.bugfix b/changes/7153.bugfix new file mode 100644 index 00000000000..7f9b68865ce --- /dev/null +++ b/changes/7153.bugfix @@ -0,0 +1 @@ +Fix the current year reference for CKAN documentation diff --git a/changes/7161.bugfix b/changes/7161.bugfix new file mode 100644 index 00000000000..ac0cce0677d --- /dev/null +++ b/changes/7161.bugfix @@ -0,0 +1 @@ +Fix bootstrap 3 webassets files to point to valid assets. \ No newline at end of file diff --git a/changes/7162.bugfix b/changes/7162.bugfix new file mode 100644 index 00000000000..96f3923619c --- /dev/null +++ b/changes/7162.bugfix @@ -0,0 +1 @@ +Fix the display of the License select element in the Dataset form. \ No newline at end of file diff --git a/changes/7163.bugfix b/changes/7163.bugfix new file mode 100644 index 00000000000..f688c5efc8d --- /dev/null +++ b/changes/7163.bugfix @@ -0,0 +1 @@ +Build CSS files with latest updates. \ No newline at end of file diff --git a/changes/7169.bugfix b/changes/7169.bugfix new file mode 100644 index 00000000000..bb4b895caed --- /dev/null +++ b/changes/7169.bugfix @@ -0,0 +1 @@ +Fix activity stream icon on Boostrap 5. Migrate activity CSS classes to the extension folder. \ No newline at end of file diff --git a/changes/7175.feature b/changes/7175.feature new file mode 100644 index 00000000000..dc5df08dbd3 --- /dev/null +++ b/changes/7175.feature @@ -0,0 +1 @@ +Added `list-unindexed` sub-commands to the `search-index` command, which lists all ununindexed packages. diff --git a/changes/7186.misc b/changes/7186.misc new file mode 100644 index 00000000000..092f6d86bdf --- /dev/null +++ b/changes/7186.misc @@ -0,0 +1 @@ +Make heading semantic in bug report template \ No newline at end of file diff --git a/changes/7187.misc b/changes/7187.misc new file mode 100644 index 00000000000..6ebd752cbe8 --- /dev/null +++ b/changes/7187.misc @@ -0,0 +1 @@ +Add title attribute to iframe \ No newline at end of file diff --git a/changes/7191.bugfix b/changes/7191.bugfix new file mode 100644 index 00000000000..3909891ecbd --- /dev/null +++ b/changes/7191.bugfix @@ -0,0 +1 @@ +Fix 404 error when selecting the same date in the changes view diff --git a/changes/7193.misc b/changes/7193.misc new file mode 100644 index 00000000000..0a7be34dc88 --- /dev/null +++ b/changes/7193.misc @@ -0,0 +1 @@ +Fix color contrast in dashboard buttons for web accesibility \ No newline at end of file diff --git a/changes/7194.misc b/changes/7194.misc new file mode 100644 index 00000000000..33bcb7bea8e --- /dev/null +++ b/changes/7194.misc @@ -0,0 +1 @@ +Make skip to content visible for keyboard-only user \ No newline at end of file diff --git a/changes/7195.misc b/changes/7195.misc new file mode 100644 index 00000000000..23ca962743f --- /dev/null +++ b/changes/7195.misc @@ -0,0 +1 @@ +Fix color contrast issue in add dataset page \ No newline at end of file diff --git a/changes/7199.changes b/changes/7199.changes new file mode 100644 index 00000000000..f485db60521 --- /dev/null +++ b/changes/7199.changes @@ -0,0 +1 @@ +Add --debug flag to compile css with sourcemaps \ No newline at end of file diff --git a/changes/7199.misc b/changes/7199.misc new file mode 100644 index 00000000000..c2ccf06245a --- /dev/null +++ b/changes/7199.misc @@ -0,0 +1 @@ +Fix color contrast of delete button in user edit page for web accesibility \ No newline at end of file diff --git a/changes/7205.bugfix b/changes/7205.bugfix new file mode 100644 index 00000000000..75f499decab --- /dev/null +++ b/changes/7205.bugfix @@ -0,0 +1 @@ +Fix display of Popular snippet. Removes old `ckan-icon` scss class. \ No newline at end of file diff --git a/changes/7217.misc b/changes/7217.misc new file mode 100644 index 00000000000..8a60129a07c --- /dev/null +++ b/changes/7217.misc @@ -0,0 +1 @@ +Add ``--quiet`` option to ``ckan user token add`` command to mak easier to integrate with automated scripts diff --git a/changes/7241.feature b/changes/7241.feature new file mode 100644 index 00000000000..72c364a5eb6 --- /dev/null +++ b/changes/7241.feature @@ -0,0 +1,3 @@ +Add new group command: `clean`. +Add `clean users` command to delete users containing images with formats +not supported in `ckan.upload.user.mimetypes` config option. \ No newline at end of file diff --git a/changes/7247.bugfix b/changes/7247.bugfix new file mode 100644 index 00000000000..4087c423fd2 --- /dev/null +++ b/changes/7247.bugfix @@ -0,0 +1 @@ +Fix icons and alignment in resource datastore tab. \ No newline at end of file diff --git a/changes/7257.feature b/changes/7257.feature new file mode 100644 index 00000000000..6c05717ae21 --- /dev/null +++ b/changes/7257.feature @@ -0,0 +1,22 @@ +`toolkit.aslist` now converts any iterable other than ``list`` and `tuple` into a ``list``: ``list(value)``. +Before, such values were just wrapped into a list, i.e: ``[value]``. + +.. list-table:: Short overview of changes + :widths: 40 30 30 + :header-rows: 1 + + * - Expresion + - Before + - After + * - ``aslist([1,2])`` + - ``[1, 2]`` + - ``[1, 2]`` + * - ``aslist({1,2})`` + - ``[{1, 2}]`` + - ``[1, 2]`` + * - ``aslist({1: "one", 2: "two"})`` + - ``[{1: "one", 2: "two"}]`` + - ``[1, 2]`` + * - ``aslist(range(1,3))`` + - ``[range(1, 3)]`` + - ``[1, 2]`` diff --git a/changes/7271.removal b/changes/7271.removal new file mode 100644 index 00000000000..f7985a7790d --- /dev/null +++ b/changes/7271.removal @@ -0,0 +1 @@ +Removes requirement-setuptools.txt since we're compatible with all recent versions of setuptools. diff --git a/changes/7344.misc b/changes/7344.misc new file mode 100644 index 00000000000..5333514aea8 --- /dev/null +++ b/changes/7344.misc @@ -0,0 +1 @@ +Updated and documented input param for `api_token_list` from `user` to `user_id`. `user` is still supported for backwards compatibility but it might be removed in the future. diff --git a/changes/7350.removal b/changes/7350.removal new file mode 100644 index 00000000000..711546903cc --- /dev/null +++ b/changes/7350.removal @@ -0,0 +1 @@ +``ckan.route_after_login`` renamed to ``ckan.auth.route_after_login`` diff --git a/changes/7351.misc b/changes/7351.misc new file mode 100644 index 00000000000..e637bff5fdf --- /dev/null +++ b/changes/7351.misc @@ -0,0 +1,10 @@ +:py:class:`~ckan.plugins.toolkit.BaseModel` class for declarative SQLAlchemy models added to :py:mod:`ckan.plugins.toolkit`. +Models extending ``BaseModel`` class are attached to the SQLAlchemy's metadata object automatically:: + + from ckan.plugins import toolkit + + class ExtModel(toolkit.BaseModel): + + __tablename__ = "ext_model" + id = Column(String(50), primary_key=True) + ... diff --git a/changes/7370.removal b/changes/7370.removal new file mode 100644 index 00000000000..55705d753a2 --- /dev/null +++ b/changes/7370.removal @@ -0,0 +1 @@ +Remove the Docker related files from the main CKAN repository. The official Docker setup can be found at the `ckan/ckan-docker `_ repository. diff --git a/changes/xloader-127.feature b/changes/xloader-127.feature new file mode 100644 index 00000000000..94a3f650c8f --- /dev/null +++ b/changes/xloader-127.feature @@ -0,0 +1 @@ +document new 'ckan.download_proxy' config value for extensions that download external URLs diff --git a/ckan-uwsgi.ini b/ckan-uwsgi.ini index 972bae48769..33995484e02 100644 --- a/ckan-uwsgi.ini +++ b/ckan-uwsgi.ini @@ -2,7 +2,7 @@ http = 127.0.0.1:8080 uid = www-data -guid = www-data +gid = www-data wsgi-file = /etc/ckan/default/wsgi.py virtualenv = /usr/lib/ckan/default module = wsgi:application @@ -13,3 +13,4 @@ max-requests = 5000 vacuum = true callable = application buffer-size = 32768 +strict = true diff --git a/ckan/__init__.py b/ckan/__init__.py index 544a5de2966..4647e7cba40 100644 --- a/ckan/__init__.py +++ b/ckan/__init__.py @@ -1,22 +1,7 @@ # encoding: utf-8 -__version__ = '2.10.0a' - -__description__ = 'CKAN Software' -__long_description__ = \ -''' -CKAN is the world's leading Open Source data portal platform. - -It powers dozens of Open Data portals around the world, including -data.gov, open.canada.ca and europeandataportal.eu but also regional, -research and community organizations. - -It makes easy to publish, share and find data online and is fully -customizable via extensions and plugins. - -Check https://ckan.org to know more. -''' -__license__ = 'AGPL' +__version__ = "2.10.0" # The packaging system relies on this import, please do not remove it -import sys; sys.path.insert(0, __path__[0]) +# type_ignore_reason: pyright thinks it's iterable +import sys; sys.path.insert(0, __path__[0]) # type: ignore diff --git a/ckan/authz.py b/ckan/authz.py index fe382db2a54..48c44084dc4 100644 --- a/ckan/authz.py +++ b/ckan/authz.py @@ -1,90 +1,99 @@ # encoding: utf-8 +from __future__ import annotations import functools -import sys +import inspect +import importlib from collections import defaultdict, OrderedDict from logging import getLogger +from typing import Any, Callable, Collection, KeysView, Optional, Union +from types import ModuleType -import six - -from ckan.common import config -from ckan.common import asbool +from ckan.common import config, current_user import ckan.plugins as p import ckan.model as model -from ckan.common import _, g +from ckan.common import _ -import ckan.lib.maintain as maintain +from ckan.types import AuthResult, AuthFunction, DataDict, Context log = getLogger(__name__) +def get_local_functions(module: ModuleType, include_private: bool = False): + """Return list of (name, func) tuples. + + Filters out all non-callables and all the items that were + imported. + """ + return inspect.getmembers( + module, + lambda func: (inspect.isfunction(func) and + inspect.getmodule(func) is module and + (include_private or not func.__name__.startswith('_')))) + + class AuthFunctions: ''' This is a private cache used by get_auth_function() and should never be accessed directly we will create an instance of it and then remove it.''' - _functions = {} + _functions: dict[str, AuthFunction] = {} - def clear(self): + def clear(self) -> None: ''' clear any stored auth functions. ''' self._functions.clear() - def keys(self): + def keys(self) -> KeysView[str]: ''' Return a list of known auth functions.''' if not self._functions: self._build() return self._functions.keys() - def get(self, function): + def get(self, function: str) -> Optional[AuthFunction]: ''' Return the requested auth function. ''' if not self._functions: self._build() return self._functions.get(function) @staticmethod - def _is_chained_auth_function(func): + def _is_chained_auth_function(func: AuthFunction) -> bool: ''' Helper function to check if a function is a chained auth function, i.e. it has been decorated with the chain auth function decorator. ''' return getattr(func, 'chained_auth_function', False) - def _build(self): - ''' Gather the auth functions. + def _build(self) -> None: + '''Gather the auth functions. + + First get the default ones in the ckan/logic/auth directory + Rather than writing them out in full will use + importlib.import_module to load anything from ckan.auth that + looks like it might be an authorisation function - First get the default ones in the ckan/logic/auth directory Rather than - writing them out in full will use __import__ to load anything from - ckan.auth that looks like it might be an authorisation function''' + ''' module_root = 'ckan.logic.auth' for auth_module_name in ['get', 'create', 'update', 'delete', 'patch']: - module_path = '%s.%s' % (module_root, auth_module_name,) - try: - module = __import__(module_path) - except ImportError: - log.debug('No auth module for action "%s"' % auth_module_name) - continue - - for part in module_path.split('.')[1:]: - module = getattr(module, part) - - for key, v in module.__dict__.items(): - if not key.startswith('_'): - # Whitelist all auth functions defined in - # logic/auth/get.py as not requiring an authorized user, - # as well as ensuring that the rest do. In both cases, do - # nothing if a decorator has already been used to define - # the behaviour - if not hasattr(v, 'auth_allow_anonymous_access'): - if auth_module_name == 'get': - v.auth_allow_anonymous_access = True - else: - v.auth_allow_anonymous_access = False - self._functions[key] = v + module = importlib.import_module( + '.' + auth_module_name, module_root) + + for key, v in get_local_functions(module): + # Whitelist all auth functions defined in + # logic/auth/get.py as not requiring an authorized user, + # as well as ensuring that the rest do. In both cases, do + # nothing if a decorator has already been used to define + # the behaviour + if not hasattr(v, 'auth_allow_anonymous_access'): + if auth_module_name == 'get': + v.auth_allow_anonymous_access = True + else: + v.auth_allow_anonymous_access = False + self._functions[key] = v # Then overwrite them with any specific ones in the plugins: - resolved_auth_function_plugins = {} + resolved_auth_function_plugins: dict[str, str] = {} fetched_auth_functions = {} chained_auth_functions = defaultdict(list) for plugin in p.PluginImplementations(p.IAuthFunctions): @@ -102,7 +111,7 @@ def _build(self): resolved_auth_function_plugins[name] = plugin.name fetched_auth_functions[name] = auth_function - for name, func_list in six.iteritems(chained_auth_functions): + for name, func_list in chained_auth_functions.items(): if (name not in fetched_auth_functions and name not in self._functions): raise Exception('The auth %r is not found for chained auth' % ( @@ -114,8 +123,13 @@ def _build(self): else: # fallback to chaining off the builtin auth function prev_func = self._functions[name] - fetched_auth_functions[name] = ( - functools.partial(func, prev_func)) + + new_func = (functools.partial(func, prev_func)) + # persisting attributes to the new partial function + for attribute, value in func.__dict__.items(): + setattr(new_func, attribute, value) + + fetched_auth_functions[name] = new_func # Use the updated ones in preference to the originals. self._functions.update(fetched_auth_functions) @@ -125,36 +139,36 @@ def _build(self): del AuthFunctions -def clear_auth_functions_cache(): +def clear_auth_functions_cache() -> None: _AuthFunctions.clear() -def auth_functions_list(): +def auth_functions_list() -> KeysView[str]: '''Returns a list of the names of the auth functions available. Currently this is to allow the Auth Audit to know if an auth function is available for a given action.''' return _AuthFunctions.keys() -def is_sysadmin(username): +def is_sysadmin(username: Optional[str]) -> bool: ''' Returns True is username is a sysadmin ''' user = _get_user(username) - return user and user.sysadmin + return bool(user and user.sysadmin) -def _get_user(username): +def _get_user(username: Optional[str]) -> Optional['model.User']: ''' - Try to get the user from g, if possible. + Try to get the user from current_user proxy, if possible. If not fallback to using the DB ''' if not username: return None # See if we can get the user without touching the DB try: - if g.userobj and g.userobj.name == username: - return g.userobj + if current_user.name == username: + return current_user # type: ignore except AttributeError: - # g.userobj not set + # current_user is anonymous pass except TypeError: # c is not available (py2) @@ -167,26 +181,31 @@ def _get_user(username): return model.User.get(username) -def get_group_or_org_admin_ids(group_id): +def get_group_or_org_admin_ids(group_id: Optional[str]) -> list[str]: if not group_id: return [] - group_id = model.Group.get(group_id).id - q = model.Session.query(model.Member) \ - .filter(model.Member.group_id == group_id) \ + group = model.Group.get(group_id) + if not group: + return [] + q = model.Session.query(model.Member.table_id) \ + .filter(model.Member.group_id == group.id) \ .filter(model.Member.table_name == 'user') \ .filter(model.Member.state == 'active') \ .filter(model.Member.capacity == 'admin') - return [a.table_id for a in q.all()] + + # type_ignore_reason: all stored memerships have table_id + return [a.table_id for a in q] -def is_authorized_boolean(action, context, data_dict=None): +def is_authorized_boolean(action: str, context: Context, data_dict: Optional[DataDict]=None) -> bool: ''' runs the auth function but just returns True if allowed else False ''' outcome = is_authorized(action, context, data_dict=data_dict) return outcome.get('success', False) -def is_authorized(action, context, data_dict=None): +def is_authorized(action: str, context: Context, + data_dict: Optional[DataDict]=None) -> AuthResult: if context.get('ignore_auth'): return {'success': True} @@ -211,51 +230,47 @@ def is_authorized(action, context, data_dict=None): # access straight away if not getattr(auth_function, 'auth_allow_anonymous_access', False) \ and not context.get('auth_user_obj'): + if isinstance(auth_function, functools.partial): + name = auth_function.func.__name__ + else: + name = auth_function.__name__ return { 'success': False, - 'msg': 'Action {0} requires an authenticated user'.format( - (auth_function if not isinstance(auth_function, functools.partial) - else auth_function.func).__name__) + 'msg': 'Action {0} requires an authenticated user'.format(name) } - return auth_function(context, data_dict) + return auth_function(context, data_dict or {}) else: raise ValueError(_('Authorization function not found: %s' % action)) # these are the permissions that roles have -ROLE_PERMISSIONS = OrderedDict([ +ROLE_PERMISSIONS: dict[str, list[str]] = OrderedDict([ ('admin', ['admin', 'membership']), - ('editor', ['read', 'delete_dataset', 'create_dataset', 'update_dataset', 'manage_group']), + ('editor', ['read', 'delete_dataset', 'create_dataset', + 'update_dataset', 'manage_group']), ('member', ['read', 'manage_group']), ]) -def get_collaborator_capacities(): +def get_collaborator_capacities() -> Collection[str]: if check_config_permission('allow_admin_collaborators'): return ('admin', 'editor', 'member') else: return ('editor', 'member') - -def _trans_role_admin(): - return _('Admin') - - -def _trans_role_editor(): - return _('Editor') - - -def _trans_role_member(): - return _('Member') +_trans_functions: dict[str, Callable[[], str]] = { + 'admin': lambda: _('Admin'), + 'editor': lambda: _('Editor'), + 'member': lambda: _('Member'), +} -def trans_role(role): - module = sys.modules[__name__] - return getattr(module, '_trans_role_%s' % role)() +def trans_role(role: str) -> str: + return _trans_functions[role]() -def roles_list(): +def roles_list() -> list[dict[str, str]]: ''' returns list of roles for forms ''' roles = [] for role in ROLE_PERMISSIONS: @@ -263,7 +278,7 @@ def roles_list(): return roles -def roles_trans(): +def roles_trans() -> dict[str, str]: ''' return dict of roles with translation ''' roles = {} for role in ROLE_PERMISSIONS: @@ -271,7 +286,7 @@ def roles_trans(): return roles -def get_roles_with_permission(permission): +def get_roles_with_permission(permission: str) -> list[str]: ''' returns the roles with the permission requested ''' roles = [] for role in ROLE_PERMISSIONS: @@ -281,7 +296,9 @@ def get_roles_with_permission(permission): return roles -def has_user_permission_for_group_or_org(group_id, user_name, permission): +def has_user_permission_for_group_or_org(group_id: Optional[str], + user_name: Optional[str], + permission: str) -> bool: ''' Check if the user has the given permissions for the group, allowing for sysadmin rights and permission cascading down a group hierarchy. @@ -302,9 +319,11 @@ def has_user_permission_for_group_or_org(group_id, user_name, permission): return False if _has_user_permission_for_groups(user_id, permission, [group_id]): return True + capacities = check_config_permission('roles_that_cascade_to_sub_groups') + assert isinstance(capacities, list) # Handle when permissions cascade. Check the user's roles on groups higher # in the group hierarchy for permission. - for capacity in check_config_permission('roles_that_cascade_to_sub_groups'): + for capacity in capacities: parent_groups = group.get_parent_group_hierarchy(type=group.type) group_ids = [group_.id for group_ in parent_groups] if _has_user_permission_for_groups(user_id, permission, group_ids, @@ -313,8 +332,9 @@ def has_user_permission_for_group_or_org(group_id, user_name, permission): return False -def _has_user_permission_for_groups(user_id, permission, group_ids, - capacity=None): +def _has_user_permission_for_groups( + user_id: str, permission: str, group_ids: list[str], + capacity: Optional[str]=None) -> bool: ''' Check if the user has the given permissions for the particular group (ignoring permissions cascading in a group hierarchy). Can also be filtered by a particular capacity. @@ -322,47 +342,53 @@ def _has_user_permission_for_groups(user_id, permission, group_ids, if not group_ids: return False # get any roles the user has for the group - q = model.Session.query(model.Member) \ - .filter(model.Member.group_id.in_(group_ids)) \ - .filter(model.Member.table_name == 'user') \ - .filter(model.Member.state == 'active') \ - .filter(model.Member.table_id == user_id) + q: Any = (model.Session.query(model.Member.capacity) + # type_ignore_reason: attribute has no method + .filter(model.Member.group_id.in_(group_ids)) # type: ignore + .filter(model.Member.table_name == 'user') + .filter(model.Member.state == 'active') + .filter(model.Member.table_id == user_id)) + if capacity: q = q.filter(model.Member.capacity == capacity) # see if any role has the required permission # admin permission allows anything for the group - for row in q.all(): + for row in q: perms = ROLE_PERMISSIONS.get(row.capacity, []) if 'admin' in perms or permission in perms: return True return False -def users_role_for_group_or_org(group_id, user_name): +def users_role_for_group_or_org( + group_id: Optional[str], user_name: Optional[str]) -> Optional[str]: ''' Returns the user's role for the group. (Ignores privileges that cascade in a group hierarchy.) ''' if not group_id: return None - group_id = model.Group.get(group_id).id + group = model.Group.get(group_id) + if not group: + return None user_id = get_user_id_for_username(user_name, allow_none=True) if not user_id: return None # get any roles the user has for the group - q = model.Session.query(model.Member) \ - .filter(model.Member.group_id == group_id) \ + q: Any = model.Session.query(model.Member.capacity) \ + .filter(model.Member.group_id == group.id) \ .filter(model.Member.table_name == 'user') \ .filter(model.Member.state == 'active') \ .filter(model.Member.table_id == user_id) # return the first role we find - for row in q.all(): + for row in q: return row.capacity return None -def has_user_permission_for_some_org(user_name, permission): +def has_user_permission_for_some_org( + user_name: Optional[str], permission: str) -> bool: ''' Check if the user has the given permission for any organization. ''' user_id = get_user_id_for_username(user_name, allow_none=True) if not user_id: @@ -372,28 +398,33 @@ def has_user_permission_for_some_org(user_name, permission): if not roles: return False # get any groups the user has with the needed role - q = model.Session.query(model.Member) \ - .filter(model.Member.table_name == 'user') \ - .filter(model.Member.state == 'active') \ - .filter(model.Member.capacity.in_(roles)) \ - .filter(model.Member.table_id == user_id) + q: Any = (model.Session.query(model.Member.group_id) + .filter(model.Member.table_name == 'user') + .filter(model.Member.state == 'active') + # type_ignore_reason: attribute has no method + .filter(model.Member.capacity.in_(roles)) # type: ignore + .filter(model.Member.table_id == user_id)) group_ids = [] - for row in q.all(): + for row in q: group_ids.append(row.group_id) # if not in any groups has no permissions if not group_ids: return False # see if any of the groups are orgs - q = model.Session.query(model.Group) \ - .filter(model.Group.is_organization == True) \ - .filter(model.Group.state == 'active') \ - .filter(model.Group.id.in_(group_ids)) + permission_exists: bool = model.Session.query( + model.Session.query(model.Group) + .filter(model.Group.is_organization == True) + .filter(model.Group.state == 'active') + # type_ignore_reason: attribute has no method + .filter(model.Group.id.in_(group_ids)).exists() # type: ignore + ).scalar() - return bool(q.count()) + return permission_exists -def get_user_id_for_username(user_name, allow_none=False): +def get_user_id_for_username( + user_name: Optional[str], allow_none: bool = False) -> Optional[str]: ''' Helper function to get user id ''' # first check if we have the user object already and get from there user = _get_user(user_name) @@ -404,7 +435,7 @@ def get_user_id_for_username(user_name, allow_none=False): raise Exception('Not logged in user') -def can_manage_collaborators(package_id, user_id): +def can_manage_collaborators(package_id: str, user_id: str) -> bool: ''' Returns True if a user is allowed to manage the collaborators of a given dataset. @@ -420,7 +451,8 @@ def can_manage_collaborators(package_id, user_id): and :ref:`ckan.auth.create_unowned_dataset`) ''' pkg = model.Package.get(package_id) - + if not pkg: + return False owner_org = pkg.owner_org if (not owner_org @@ -439,7 +471,9 @@ def can_manage_collaborators(package_id, user_id): return user_is_collaborator_on_dataset(user_id, pkg.id, 'admin') -def user_is_collaborator_on_dataset(user_id, dataset_id, capacity=None): +def user_is_collaborator_on_dataset( + user_id: str, dataset_id: str, + capacity: Optional[Union[str, list[str]]] = None) -> bool: ''' Returns True if the provided user is a collaborator on the provided dataset. @@ -455,14 +489,15 @@ def user_is_collaborator_on_dataset(user_id, dataset_id, capacity=None): .filter(model.PackageMember.package_id == dataset_id) if capacity: - if isinstance(capacity, six.string_types): + if isinstance(capacity, str): capacity = [capacity] - q = q.filter(model.PackageMember.capacity.in_(capacity)) + # type_ignore_reason: attribute has no method + q = q.filter(model.PackageMember.capacity.in_(capacity)) # type: ignore - return q.count() > 0 + return model.Session.query(q.exists()).scalar() -CONFIG_PERMISSIONS_DEFAULTS = { +CONFIG_PERMISSIONS_DEFAULTS: dict[str, Union[bool, str]] = { # permission and default # these are prefixed with ckan.auth. in config to override 'anon_create_dataset': False, @@ -483,7 +518,7 @@ def user_is_collaborator_on_dataset(user_id, dataset_id, capacity=None): } -def check_config_permission(permission): +def check_config_permission(permission: str) -> Union[list[str], bool]: '''Returns the configuration value for the provided permission Permission is a string indentifying the auth permission (eg @@ -503,37 +538,14 @@ def check_config_permission(permission): if key not in CONFIG_PERMISSIONS_DEFAULTS: return False - default_value = CONFIG_PERMISSIONS_DEFAULTS.get(key) - config_key = 'ckan.auth.' + key - value = config.get(config_key, default_value) - - if key == 'roles_that_cascade_to_sub_groups': - # This permission is set as a list of strings (space separated) - value = value.split() if value else [] - else: - value = asbool(value) + value = config.get(config_key) return value -@maintain.deprecated('Use auth_is_loggedin_user instead') -def auth_is_registered_user(): - ''' - This function is deprecated, please use the auth_is_loggedin_user instead - ''' - return auth_is_loggedin_user() - -def auth_is_loggedin_user(): - ''' Do we have a logged in user ''' - try: - context_user = g.user - except TypeError: - context_user = None - return bool(context_user) - -def auth_is_anon_user(context): +def auth_is_anon_user(context: Context) -> bool: ''' Is this an anonymous user? eg Not logged in if a web request and not user defined in context if logic functions called directly diff --git a/ckan/cli/__init__.py b/ckan/cli/__init__.py index 5d342005604..9922fb233f0 100644 --- a/ckan/cli/__init__.py +++ b/ckan/cli/__init__.py @@ -1,82 +1,129 @@ # encoding: utf-8 +from __future__ import annotations -import sys import os +from typing import Any, Optional import click import logging from logging.config import fileConfig as loggingFileConfig -from six.moves.configparser import ConfigParser +from configparser import ConfigParser, RawConfigParser from ckan.exceptions import CkanConfigurationException +from ckan.types import Config log = logging.getLogger(__name__) class CKANConfigLoader(object): - def __init__(self, filename): + config: Config + config_file: str + parser: ConfigParser + section: str + + def __init__(self, filename: str) -> None: self.config_file = filename.strip() self.config = dict() self.parser = ConfigParser() + # Preserve case in config keys + self.parser.optionxform = lambda optionstr: str(optionstr) self.section = u'app:main' - defaults = {u'__file__': os.path.abspath(self.config_file)} + defaults = dict( + (k, v) for k, v in os.environ.items() + if k.startswith("CKAN_")) + defaults['__file__'] = os.path.abspath(self.config_file) self._update_defaults(defaults) self._create_config_object() - def _update_defaults(self, new_defaults): + def _update_defaults(self, new_defaults: dict[str, Any]) -> None: for key, value in new_defaults.items(): - self.parser._defaults[key] = value + # type_ignore_reason: using implementation details + self.parser._defaults[key] = value # type: ignore - def _read_config_file(self, filename): + def _read_config_file(self, filename: str) -> None: defaults = {u'here': os.path.dirname(os.path.abspath(filename))} self._update_defaults(defaults) self.parser.read(filename) - def _update_config(self): + def _update_config(self) -> None: options = self.parser.options(self.section) for option in options: - if option not in self.config or option in self.parser.defaults(): - value = self.parser.get(self.section, option) - self.config[option] = value - if option in self.parser.defaults(): - self.config[u'global_conf'][option] = value + value = self.parser.get(self.section, option) + self.config[option] = value + + # eager interpolation of the `here` variable. Otherwise it will get + # shadowed by the higher-level config file. + raw = self.parser.get(self.section, option, raw=True) + if "%(here)s" in raw: + self.parser.set(self.section, option, value) + + def _unwrap_config_chain(self, filename: str) -> list[str]: + """Get all names of files in use-chain. + + Parse files using RawConfigParser, because top-level config file can + use variaables from the lower-level config files, which are not + initialized yet. + """ + parser = RawConfigParser() + chain = [] + while True: + parser.read(filename) + chain.append(filename) + use = parser.get(self.section, "use") + if not use: + return chain + try: + schema, next_config = use.split(":", 1) + except ValueError: + raise CkanConfigurationException( + "Missing colon symbol in the value of `use` " + + f"option inside {filename}: {use}" + ) + + if schema != "config": + return chain + filename = os.path.join( + os.path.dirname(os.path.abspath(filename)), next_config) + if filename in chain: + joined_chain = ' -> '.join(chain + [filename]) + raise CkanConfigurationException( + 'Circular dependency located in ' + f'the configuration chain: {joined_chain}' + ) def _create_config_object(self): - self._read_config_file(self.config_file) - - # # The global_config key is to keep compatibility with Pylons. - # # It can be safely removed when the Flask migration is completed. - self.config[u'global_conf'] = self.parser.defaults().copy() - - self._update_config() - - schema, path = self.parser.get(self.section, u'use').split(u':') - if schema == u'config': - use_config_path = os.path.join( - os.path.dirname(os.path.abspath(self.config_file)), path) - self._read_config_file(use_config_path) + chain = self._unwrap_config_chain(self.config_file) + for filename in reversed(chain): + self._read_config_file(filename) self._update_config() + log.debug( + u'Loaded configuration from the following files: %s', + chain + ) - def get_config(self): + def get_config(self) -> Config: return self.config.copy() -def error_shout(exception): +def error_shout(exception: Any) -> None: + """Report CLI error with a styled message. + """ click.secho(str(exception), fg=u'red', err=True) -def load_config(ini_path=None): +def load_config(ini_path: Optional[str] = None) -> Config: if ini_path: if ini_path.startswith(u'~'): ini_path = os.path.expanduser(ini_path) - filename = os.path.abspath(ini_path) - config_source = u'-c parameter' + filename: Optional[str] = os.path.abspath(ini_path) + config_source = [u'-c parameter'] elif os.environ.get(u'CKAN_INI'): - filename = os.environ.get(u'CKAN_INI') - config_source = u'$CKAN_INI' + filename = os.environ[u'CKAN_INI'] + config_source = [u'$CKAN_INI'] else: # deprecated method since CKAN 2.9 default_filenames = [u'ckan.ini', u'development.ini'] + config_source = default_filenames filename = None for default_filename in default_filenames: check_file = os.path.join(os.getcwd(), default_filename) @@ -93,7 +140,7 @@ def load_config(ini_path=None): msg = msg.format(u', '.join(default_filenames)) raise CkanConfigurationException(msg) - if not os.path.exists(filename): + if not filename or not os.path.exists(filename): msg = u'Config file not found: %s' % filename msg += u'\n(Given by: %s)' % config_source raise CkanConfigurationException(msg) diff --git a/ckan/cli/asset.py b/ckan/cli/asset.py index 449c349924e..0cc33e7e18d 100644 --- a/ckan/cli/asset.py +++ b/ckan/cli/asset.py @@ -12,10 +12,9 @@ log = logging.getLogger(__name__) -@click.group() +@click.group(short_help=u"WebAssets commands.") def asset(): """WebAssets commands. - """ pass diff --git a/ckan/cli/clean.py b/ckan/cli/clean.py new file mode 100644 index 00000000000..dc4e0addc6a --- /dev/null +++ b/ckan/cli/clean.py @@ -0,0 +1,94 @@ +# encoding: utf-8 + +import click +import magic +import os + +from typing import List + +from ckan import model +from ckan import logic +from ckan.common import config +from ckan.lib.uploader import get_uploader +from ckan.types import Context + + +@click.group(short_help="Provide commands to clean entities from the database") +@click.help_option("-h", "--help") +def clean(): + pass + + +def _get_users_with_invalid_image(mimetypes: List[str]) -> List[model.User]: + """Returns a list of users containing images with mimetypes not supported. + """ + users = model.User.all() + users_with_img = [u for u in users if u.image_url] + invalid = [] + for user in users_with_img: + upload = get_uploader("user", old_filename=user.image_url) + filepath = upload.old_filepath # type: ignore + if os.path.exists(filepath): + mimetype = magic.from_file(filepath, mime=True) + if mimetype not in mimetypes: + invalid.append(user) + return invalid + + +@clean.command("users", short_help="Clean users containing invalid images.") +@click.option( + "-f", "--force", is_flag=True, help="Do not ask for comfirmation." +) +def users(force: bool): + """Removes users with invalid images from the database. + + Invalid images are the ones with mimetypes not defined in + `ckan.upload.user.mimetypes` configuration option. + + This command will work only for CKAN's default Upload, other + extensions defining upload interfaces will need to implement its + own logic to retrieve and determine if an uploaded image contains + an invalid mimetype. + + Example: + + ckan clean users + ckan clean users --force + + """ + mimetypes = config.get("ckan.upload.user.mimetypes") + if not mimetypes: + click.echo("No mimetypes have been configured for user uploads.") + return + + invalid = _get_users_with_invalid_image(mimetypes) + + if not invalid: + click.echo("No users were found with invalid images.") + return + + for user in invalid: + msg = "User {} has an invalid image: {}".format( + user.name, user.image_url + ) + click.echo(msg) + + if not force: + click.confirm("Permanently delete users and their images?", abort=True) + + site_user = logic.get_action("get_site_user")({"ignore_auth": True}, {}) + context: Context = {"user": site_user["name"]} + + for user in invalid: + upload = get_uploader("user", old_filename=user.image_url) + file_path = upload.old_filepath # type: ignore + try: + os.remove(file_path) + except Exception: + msg = "Cannot remove {}. User will not be deleted.".format( + file_path + ) + click.echo(msg) + else: + logic.get_action("user_delete")(context, {"id": user.name}) + click.secho("Deleted user: %s" % user.name, fg="green", bold=True) diff --git a/ckan/cli/cli.py b/ckan/cli/cli.py index cc61b978341..ae086f97051 100644 --- a/ckan/cli/cli.py +++ b/ckan/cli/cli.py @@ -1,10 +1,11 @@ # encoding: utf-8 +from __future__ import annotations import logging from collections import defaultdict +from typing import Optional from pkg_resources import iter_entry_points -import six import click import sys @@ -12,27 +13,31 @@ import ckan.cli as ckan_cli from ckan.config.middleware import make_app from ckan.exceptions import CkanConfigurationException -from ckan.cli import ( - config_tool, - jobs, - front_end_build, +from . import ( + asset, + config, + clean, + dataset, db, search_index, server, + generate, + jobs, + notify, + plugin_info, profile, - asset, + sass, sysadmin, + tracking, translation, - dataset, + user, views, - plugin_info, - notify, - tracking, - minify, - less, - generate, - user + config_tool, + error_shout, + shell ) -from ckan.cli import seed +META_ATTR = u'_ckan_meta' +CMD_TYPE_PLUGIN = u'plugin' +CMD_TYPE_ENTRY = u'entry_point' log = logging.getLogger(__name__) @@ -43,100 +48,37 @@ ] -class CkanCommand(object): +class CtxObject(object): - def __init__(self, conf=None): + def __init__(self, conf: Optional[str] = None): # Don't import `load_config` by itself, rather call it using # module so that it can be patched during tests - self.config = ckan_cli.load_config(conf) - self.app = make_app(self.config) - - -def _get_commands_from_plugins(plugins): - for plugin in plugins: - for cmd in plugin.get_commands(): - cmd._ckan_meta = { - u'name': plugin.name, - u'type': u'plugin' - } - yield cmd - - -def _get_commands_from_entry_point(entry_point=u'ckan.click_command'): - registered_entries = {} - for entry in iter_entry_points(entry_point): - if entry.name in registered_entries: - p.toolkit.error_shout(( - u'Attempt to override entry_point `{name}`.\n' - u'First encounter:\n\t{first!r}\n' - u'Second encounter:\n\t{second!r}\n' - u'Either uninstall one of mentioned extensions or update' - u' corresponding `setup.py` and re-install the extension.' - ).format( - name=entry.name, - first=registered_entries[entry.name].dist, - second=entry.dist)) - raise click.Abort() - registered_entries[entry.name] = entry + raw_config = ckan_cli.load_config(conf) + self.app = make_app(raw_config) - cmd = entry.load() - cmd._ckan_meta = { - u'name': entry.name, - u'type': u'entry_point' - } - yield cmd - - -def _init_ckan_config(ctx, param, value): - is_help = u'--help' in sys.argv - no_config = False - if len(sys.argv) > 1: - for cmd in _no_config_commands: - if sys.argv[1:len(cmd) + 1] == cmd: - no_config = True - break - if no_config or is_help: - return + # Attach the actual CKAN config object to the context + from ckan.common import config + self.config = config - try: - ctx.obj = CkanCommand(value) - except CkanConfigurationException as e: - p.toolkit.error_shout(e) - raise click.Abort() - if six.PY2: - ctx.meta["flask_app"] = ctx.obj.app.apps["flask_app"]._wsgi_app - else: - ctx.meta["flask_app"] = ctx.obj.app._wsgi_app - - for cmd in _get_commands_from_entry_point(): - ctx.command.add_command(cmd) - - plugins = p.PluginImplementations(p.IClick) - for cmd in _get_commands_from_plugins(plugins): - ctx.command.add_command(cmd) - - -click_config_option = click.option( - u'-c', - u'--config', - default=None, - metavar=u'CONFIG', - help=u'Config file to use (default: development.ini)', - is_eager=True, - callback=_init_ckan_config -) - - -class CustomGroup(click.Group): +class ExtendableGroup(click.Group): _section_titles = { - u'plugin': u'Plugins', - u'entry_point': u'Entry points', + CMD_TYPE_PLUGIN: u'Plugins', + CMD_TYPE_ENTRY: u'Entry points', } - def format_commands(self, ctx, formatter): - # Without any arguments click skips option callbacks. - self.parse_args(ctx, [u'help']) + def format_commands( + self, ctx: click.Context, formatter: click.HelpFormatter): + """Print help message. + + Includes information about commands that were registered by extensions. + """ + # click won't parse config file from envvar if no other options + # provided, except for `--help`. In this case it has to be done + # manually. + if not ctx.obj: + _add_ctx_object(ctx) + _add_external_commands(ctx) commands = [] ext_commands = defaultdict(lambda: defaultdict(list)) @@ -145,9 +87,11 @@ def format_commands(self, ctx, formatter): cmd = self.get_command(ctx, subcommand) if cmd is None: continue + if cmd.hidden: + continue help = cmd.short_help or u'' - meta = getattr(cmd, u'_ckan_meta', None) + meta = getattr(cmd, META_ATTR, None) if meta: ext_commands[meta[u'type']][meta[u'name']].append( (subcommand, help)) @@ -160,34 +104,134 @@ def format_commands(self, ctx, formatter): for section, group in ext_commands.items(): with formatter.section(self._section_titles.get(section, section)): - for _ext, rows in group.items(): + for rows in group.values(): formatter.write_dl(rows) + def parse_args(self, ctx: click.Context, args: list[str]): + """Preprocess options and arguments. + + As long as at least one option is provided, click won't fallback to + printing help message. That means that `ckan -c config.ini` will be + executed as command, instead of just printing help message(as `ckan -c + config.ini --help`). + In order to fix it, we have to check whether there is at least one + argument. If no, let's print help message manually + + """ + result = super().parse_args(ctx, args) + if not ctx.protected_args and not ctx.args: + click.echo(ctx.get_help(), color=ctx.color) + ctx.exit() + return result + + +def _init_ckan_config(ctx: click.Context, param: str, value: str): + if any(sys.argv[1:len(cmd) + 1] == cmd for cmd in _no_config_commands): + return + _add_ctx_object(ctx, value) + _add_external_commands(ctx) + + +def _add_ctx_object(ctx: click.Context, path: Optional[str] = None): + """Initialize CKAN App using config file available under provided path. -@click.group(cls=CustomGroup) + """ + try: + ctx.obj = CtxObject(path) + except CkanConfigurationException as e: + error_shout(e) + ctx.abort() + + ctx.meta["flask_app"] = ctx.obj.app._wsgi_app + + # Remove all commands that were registered by extensions before + # adding new ones. Such situation is possible only during tests, + # because we are using singleton as main entry point, so it + # preserves its state even between tests + commands = getattr(ctx.command, "commands") + for key, cmd in list(commands.items()): + if hasattr(cmd, META_ATTR): + commands.pop(key) + + +def _add_external_commands(ctx: click.Context): + add = getattr(ctx.command, "add_command") + for cmd in _get_commands_from_entry_point(): + add(cmd) + + plugins = p.PluginImplementations(p.IClick) + for cmd in _get_commands_from_plugins(plugins): + add(cmd) + + +def _command_with_ckan_meta(cmd: click.Command, name: str, type_: str): + """Mark command as one retrived from CKAN extension. + + This information is used when CLI help text is generated. + """ + setattr(cmd, META_ATTR, {u'name': name, u'type': type_}) + return cmd + + +def _get_commands_from_plugins(plugins: p.PluginImplementations[p.IClick]): + """Register commands that are available when plugin enabled. + + """ + for plugin in plugins: + for cmd in plugin.get_commands(): + yield _command_with_ckan_meta(cmd, plugin.name, CMD_TYPE_PLUGIN) + + +def _get_commands_from_entry_point(entry_point: str = 'ckan.click_command'): + """Register commands that are available even if plugin is not enabled. + + """ + registered_entries = {} + for entry in iter_entry_points(entry_point): + if entry.name in registered_entries: + error_shout(( + u'Attempt to override entry_point `{name}`.\n' + u'First encounter:\n\t{first!r}\n' + u'Second encounter:\n\t{second!r}\n' + u'Either uninstall one of mentioned extensions or update' + u' corresponding `setup.py` and re-install the extension.' + ).format( + name=entry.name, + first=registered_entries[entry.name].dist, + second=entry.dist)) + raise click.Abort() + registered_entries[entry.name] = entry + + yield _command_with_ckan_meta(entry.load(), entry.name, CMD_TYPE_ENTRY) + + +@click.group(cls=ExtendableGroup) +@click.option( + u'-c', u'--config', metavar=u'CONFIG', + is_eager=True, callback=_init_ckan_config, expose_value=False, + help=u'Config file to use (default: ckan.ini)') @click.help_option(u'-h', u'--help') -@click_config_option -def ckan(config, *args, **kwargs): +def ckan(): pass -ckan.add_command(jobs.jobs) +ckan.add_command(asset.asset) +ckan.add_command(config.config) ckan.add_command(config_tool.config_tool) -ckan.add_command(front_end_build.front_end_build) -ckan.add_command(server.run) -ckan.add_command(profile.profile) -ckan.add_command(seed.seed) +ckan.add_command(dataset.dataset) ckan.add_command(db.db) +ckan.add_command(generate.generate) +ckan.add_command(jobs.jobs) +ckan.add_command(notify.notify) +ckan.add_command(plugin_info.plugin_info) +ckan.add_command(profile.profile) +ckan.add_command(sass.sass) ckan.add_command(search_index.search_index) +ckan.add_command(server.run) ckan.add_command(sysadmin.sysadmin) -ckan.add_command(asset.asset) -ckan.add_command(translation.translation) -ckan.add_command(dataset.dataset) -ckan.add_command(views.views) -ckan.add_command(plugin_info.plugin_info) -ckan.add_command(notify.notify) ckan.add_command(tracking.tracking) -ckan.add_command(minify.minify) -ckan.add_command(less.less) -ckan.add_command(generate.generate) +ckan.add_command(translation.translation) ckan.add_command(user.user) +ckan.add_command(views.views) +ckan.add_command(shell.shell) +ckan.add_command(clean.clean) diff --git a/ckan/cli/config.py b/ckan/cli/config.py new file mode 100644 index 00000000000..362c90b4021 --- /dev/null +++ b/ckan/cli/config.py @@ -0,0 +1,232 @@ +# -*- coding: utf-8 -*- +from __future__ import annotations + +import itertools +from typing import Iterable +import click + +from ckan.config.declaration import Declaration, Flag +from ckan.config.declaration.key import Pattern +from ckan.common import config as cfg + +from . import error_shout + + +@click.group( + short_help="Search, validate and describe config options." +) +def config(): + pass + + +@config.command() +@click.argument("plugins", nargs=-1) +@click.option( + "--core", + is_flag=True, + help="Include declarations of CKAN core config options", +) +@click.option( + "--enabled", + is_flag=True, + help="Include declarations of plugins enabled in the CKAN config file", +) +@click.option( + "-f", + "--format", + "fmt", + type=click.Choice(["python", "yaml", "dict", "json", "toml"]), + default="python", + help="Output the config declaration in this format", +) +def describe(plugins: tuple[str, ...], core: bool, enabled: bool, fmt: str): + """Print out config declarations for the given plugins.""" + decl = _declaration(plugins, core, enabled) + if decl: + click.echo(decl.describe(fmt)) + + +@config.command() +@click.argument("plugins", nargs=-1) +@click.option( + "--core", + is_flag=True, + help="Include declarations of CKAN core config options", +) +@click.option( + "--enabled", + is_flag=True, + help="Include declarations of plugins enabled in the CKAN config file", +) +@click.option( + "-d", + "--include-docs", + is_flag=True, + help="Include documentation for options", +) +@click.option( + "-m", + "--minimal", + is_flag=True, + help="Print only mandatory options", +) +def declaration( + plugins: tuple[str, ...], + core: bool, + enabled: bool, + include_docs: bool, + minimal: bool, +): + """Print declared config options for the given plugins.""" + + decl = _declaration(plugins, core, enabled) + if decl: + click.echo(decl.into_ini(minimal, include_docs)) + + +@config.command() +@click.argument("pattern", default="*") +@click.option( + "-i", + "--include-plugin", + "plugins", + multiple=True, + help="Include this plugin even if disabled", +) +@click.option( + "--with-default", + is_flag=True, + help="Print default value of the config option", +) +@click.option( + "--with-current", + is_flag=True, + help="Print an actual value of the config option", +) +@click.option( + "--custom-only", + is_flag=True, + help="Ignore options that are using default value", +) +@click.option( + "--no-custom", + is_flag=True, + help="Ignore options that are not using default value", +) +@click.option( + "--explain", is_flag=True, help="Print documentation for config option" +) +def search( + pattern: str, + plugins: tuple[str, ...], + with_default: bool, + with_current: bool, + custom_only: bool, + no_custom: bool, + explain: bool, +): + """Print all declared config options that match pattern.""" + decl = _declaration(plugins, True, True) + + for key in decl.iter_options(pattern=pattern): + if isinstance(key, Pattern): + continue + option = decl[key] + default = option.default + current = option.normalize(cfg.get(str(key), default)) + if no_custom and default != current: + continue + if custom_only and default == current: + continue + + default_section = "" + current_section = "" + if with_default: + default_section = click.style( + f" [Default: {repr(default)}]", fg="red" + ) + if with_current: + current_section = click.style( + f" [Current: {repr(current)}]", fg="green" + ) + docs = "" + if explain and option.description: + lines = option.description.splitlines() + lines += ["", f"Default value: {repr(default)}"] + if option.example: + lines += ["", f"Example: {key} = {option.example}"] + docs = "\n".join(f"\t{dl}" for dl in lines) + docs = click.style(f"\n{docs}\n", bold=True) + + line = f"{key}{default_section}{current_section}{docs}" + click.secho(line) + + +@config.command() +@click.option( + "-i", + "--include-plugin", + "plugins", + multiple=True, + help="Include this plugin even if disabled", +) +def undeclared(plugins: tuple[str, ...]): + """Print config options that have no declaration. + + This command includes options from the config file as well as options set + in run-time, by IConfigurer, for example. + + """ + decl = _declaration(plugins, True, True) + + declared = set(decl.iter_options(exclude=Flag.none())) + patterns = {key for key in declared if isinstance(key, Pattern)} + declared -= patterns + available = set(cfg) + + undeclared = { + s + for s in available.difference(declared) + if not any(s == p for p in patterns) + } + + for key in undeclared: + click.echo(key) + + +@config.command() +@click.option( + "-i", + "--include-plugin", + "plugins", + multiple=True, + help="Include this plugin even if disabled", +) +def validate(plugins: tuple[str, ...]): + """Validate the global configuration object against the declaration.""" + decl = _declaration(plugins, True, True) + _, errors = decl.validate(cfg) + + for name, errors in errors.items(): + click.secho(name, bold=True) + for error in errors: + error_shout("\t" + error) + + +def _declaration( + plugins: Iterable[str], include_core: bool, include_enabled: bool +) -> Declaration: + decl = Declaration() + if include_core: + decl.load_core_declaration() + + additional = () + if include_enabled: + additional = ( + p for p in cfg.get("ckan.plugins") if p not in plugins + ) + + for name in itertools.chain(additional, plugins): + decl.load_plugin(name) + + return decl diff --git a/ckan/cli/config_tool.py b/ckan/cli/config_tool.py index ebf3e81fe4a..0910464a827 100644 --- a/ckan/cli/config_tool.py +++ b/ckan/cli/config_tool.py @@ -1,11 +1,11 @@ # encoding: utf-8 +from __future__ import annotations import logging - import click -from ckan.cli import error_shout import ckan.lib.config_tool as ct +from ckan.cli import error_shout log = logging.getLogger(__name__) @@ -13,7 +13,7 @@ class ConfigOption(click.ParamType): name = u'config-option' - def convert(self, value, param, ctx): + def convert(self, value: str, param: str, ctx: click.Context): if u'=' not in value: self.fail( u'An option does not have an equals sign. ' @@ -47,7 +47,10 @@ def convert(self, value, param, ctx): ) @click.argument(u'config_filepath', type=click.Path(exists=True)) @click.argument(u'options', nargs=-1, type=ConfigOption()) -def config_tool(config_filepath, options, section, edit, merge_filepath): +def config_tool( + config_filepath: str, + options: list[str], section: str, edit: bool, + merge_filepath: str) -> None: u'''Tool for editing options in a CKAN config file ckan config-tool = [= ...] diff --git a/ckan/cli/dataset.py b/ckan/cli/dataset.py index e22493cb0f9..59111a4d0e5 100644 --- a/ckan/cli/dataset.py +++ b/ckan/cli/dataset.py @@ -4,23 +4,25 @@ import pprint import click -from six import text_type + import ckan.logic as logic import ckan.model as model +from ckan.types import Context log = logging.getLogger(__name__) -@click.group() +@click.group(short_help=u"Manage datasets") def dataset(): - u'''Manage datasets - ''' + """Manage datasets. + """ + pass @dataset.command() @click.argument(u'package') -def show(package): +def show(package: str): u'''Shows dataset properties. ''' dataset = _get_dataset(package) @@ -47,7 +49,7 @@ def list(): @dataset.command() @click.argument(u'package') -def delete(package): +def delete(package: str): u'''Changes dataset state to 'deleted'. ''' dataset = _get_dataset(package) @@ -66,20 +68,20 @@ def delete(package): @dataset.command() @click.argument(u'package') -def purge(package): +def purge(package: str): u'''Removes dataset from db entirely. ''' dataset = _get_dataset(package) name = dataset.name site_user = logic.get_action(u'get_site_user')({u'ignore_auth': True}, {}) - context = {u'user': site_user[u'name'], u'ignore_auth': True} + context: Context = {u'user': site_user[u'name'], u'ignore_auth': True} logic.get_action(u'dataset_purge')(context, {u'id': package}) click.echo(u'%s purged' % name) -def _get_dataset(package): - dataset = model.Package.get(text_type(package)) +def _get_dataset(package: str): + dataset = model.Package.get(str(package)) assert dataset, u'Could not find dataset matching reference: {}'.format( package ) diff --git a/ckan/cli/db.py b/ckan/cli/db.py index 98a54a06dae..2b14fde25e7 100644 --- a/ckan/cli/db.py +++ b/ckan/cli/db.py @@ -1,23 +1,27 @@ # encoding: utf-8 +from __future__ import annotations import inspect import logging import os +import contextlib +from typing import Optional import click from itertools import groupby import ckan.migration as migration_repo import ckan.plugins as p -import ckan.plugins.toolkit as tk import ckan.model as model +from ckan.common import config +from . import error_shout log = logging.getLogger(__name__) applies_to_plugin = click.option(u"-p", u"--plugin", help=u"Affected plugin.") -@click.group() +@click.group(short_help=u"Database management commands.") def db(): """Database management commands. """ @@ -30,10 +34,9 @@ def init(): """ log.info(u"Initialize the Database") try: - import ckan.model as model model.repo.init_db() except Exception as e: - tk.error_shout(e) + error_shout(e) else: click.secho(u'Initialising DB: SUCCESS', fg=u'green', bold=True) @@ -47,10 +50,9 @@ def clean(): """Clean the database. """ try: - import ckan.model as model model.repo.clean_db() except Exception as e: - tk.error_shout(e) + error_shout(e) else: click.secho(u'Cleaning DB: SUCCESS', fg=u'green', bold=True) @@ -58,45 +60,78 @@ def clean(): @db.command() @click.option(u'-v', u'--version', help=u'Migration version', default=u'head') @applies_to_plugin -def upgrade(version, plugin): +def upgrade(version: str, plugin: str): """Upgrade the database. """ - try: - import ckan.model as model - model.repo._alembic_ini = _resolve_alembic_config(plugin) - model.repo.upgrade_db(version) - except Exception as e: - tk.error_shout(e) - else: - click.secho(u'Upgrading DB: SUCCESS', fg=u'green', bold=True) + _run_migrations(plugin, version) + click.secho(u'Upgrading DB: SUCCESS', fg=u'green', bold=True) @db.command() @click.option(u'-v', u'--version', help=u'Migration version', default=u'base') @applies_to_plugin -def downgrade(version, plugin): +def downgrade(version: str, plugin: str): """Downgrade the database. """ - try: - import ckan.model as model - model.repo._alembic_ini = _resolve_alembic_config(plugin) - model.repo.downgrade_db(version) - except Exception as e: - tk.error_shout(e) - else: - click.secho(u'Downgrading DB: SUCCESS', fg=u'green', bold=True) + _run_migrations(plugin, version, False) + click.secho(u'Downgrading DB: SUCCESS', fg=u'green', bold=True) + + +@db.command() +@click.option("--apply", is_flag=True, help="Apply all pending migrations") +def pending_migrations(apply: bool): + """List all sources with unapplied migrations. + """ + pending = _get_pending_plugins() + if not pending: + click.secho("All plugins are up-to-date", fg="green") + for plugin, n in sorted(pending.items()): + click.secho("{n} unapplied migrations for {p}".format( + p=click.style(plugin, bold=True), + n=click.style(str(n), bold=True))) + if apply: + _run_migrations(plugin) + + +def _get_pending_plugins() -> dict[str, int]: + from alembic.command import history + plugins = [(plugin, state) + for plugin, state + in ((plugin, current_revision(plugin)) + for plugin in config.get('ckan.plugins')) + if state and not state.endswith('(head)')] + pending = {} + for plugin, current in plugins: + with _repo_for_plugin(plugin) as repo: + repo.setup_migration_version_control() + history(repo.alembic_config) + ahead = repo.take_alembic_output() + if current != 'base': + # The last revision in history describes step from void to the + # first revision. If we not on the `base`, we've already run + # this migration + ahead = ahead[:-1] + if ahead: + pending[plugin] = len(ahead) + return pending + + +def _run_migrations(plugin: str, version: str = "head", forward: bool = True): + if not version: + version = "head" if forward else "base" + with _repo_for_plugin(plugin) as repo: + if forward: + repo.upgrade_db(version) + else: + repo.downgrade_db(version) @db.command() @applies_to_plugin -def version(plugin): +def version(plugin: str): """Returns current version of data schema. """ - import ckan.model as model - model.repo._alembic_ini = _resolve_alembic_config(plugin) - log.info(u"Returning current DB version") - model.repo.setup_migration_version_control() - current = model.repo.current_version() + current = current_revision(plugin) or '' try: current = _version_hash_to_ordinal(current) except ValueError: @@ -106,6 +141,12 @@ def version(plugin): bold=True) +def current_revision(plugin: str) -> Optional[str]: + with _repo_for_plugin(plugin) as repo: + repo.setup_migration_version_control() + return repo.current_version() + + @db.command(u"duplicate_emails", short_help=u"Check users email for duplicate") def duplicate_emails(): u'''Check users email for duplicate''' @@ -117,21 +158,23 @@ def duplicate_emails(): .filter(model.User.email != u"") \ .order_by(model.User.email).all() - if not q: - log.info(u"No duplicate emails found") + duplicates_found = False try: for k, grp in groupby(q, lambda x: x[0]): users = [user[1] for user in grp] if len(users) > 1: + duplicates_found = True s = u"{} appears {} time(s). Users: {}" click.secho( s.format(k, len(users), u", ".join(users)), fg=u"green", bold=True) except Exception as e: - tk.error_shout(e) + error_shout(e) + if not duplicates_found: + click.secho(u"No duplicate emails found", fg=u"green") -def _version_hash_to_ordinal(version): +def _version_hash_to_ordinal(version: str): if u'base' == version: return 0 versions_dir = os.path.join(os.path.dirname(migration_repo.__file__), @@ -144,17 +187,19 @@ def _version_hash_to_ordinal(version): for name in versions: if version in name: return int(name.split(u'_')[0]) - tk.error_shout(u'Version `{}` was not found in {}'.format( + error_shout(u'Version `{}` was not found in {}'.format( version, versions_dir)) -def _resolve_alembic_config(plugin): +def _resolve_alembic_config(plugin: str): if plugin: plugin_obj = p.get_plugin(plugin) if plugin_obj is None: - tk.error_shout(u"Plugin '{}' cannot be loaded.".format(plugin)) + error_shout(u"Plugin '{}' cannot be loaded.".format(plugin)) raise click.Abort() - plugin_dir = os.path.dirname(inspect.getsourcefile(type(plugin_obj))) + source = inspect.getsourcefile(type(plugin_obj)) + assert source + plugin_dir = os.path.dirname(source) # if there is `plugin` folder instead of single_file, find # plugin's parent dir @@ -167,3 +212,13 @@ def _resolve_alembic_config(plugin): import ckan.migration as _cm migration_dir = os.path.dirname(_cm.__file__) return os.path.join(migration_dir, u"alembic.ini") + + +@contextlib.contextmanager +def _repo_for_plugin(plugin: str): + original = model.repo._alembic_ini + model.repo._alembic_ini = _resolve_alembic_config(plugin) + try: + yield model.repo + finally: + model.repo._alembic_ini = original diff --git a/ckan/cli/front_end_build.py b/ckan/cli/front_end_build.py deleted file mode 100644 index b6f6ec7c358..00000000000 --- a/ckan/cli/front_end_build.py +++ /dev/null @@ -1,34 +0,0 @@ -# encoding: utf-8 - -import os - -import click - -from ckan.cli import minify, less, translation -import ckan.plugins.toolkit as toolkit - - -@click.group( - name=u"front-end-build", - short_help=u"Creates and minifies css and JavaScript files.", - invoke_without_command=True, -) -@click.pass_context -def front_end_build(ctx): - if ctx.invoked_subcommand is None: - ctx.invoke(build) - - -@front_end_build.command(short_help=u"Compile css and js.",) -@click.pass_context -def build(ctx): - ctx.invoke(less.less) - ctx.invoke(translation.js) - - # minification - public = toolkit.config.get(u"ckan.base_public_folder") - root = os.path.join(os.path.dirname(__file__), u"..", public, u"base") - root = os.path.abspath(root) - ckanext = os.path.join(os.path.dirname(__file__), u"..", u"..", u"ckanext") - ckanext = os.path.abspath(ckanext) - cmd = ctx.invoke(minify.minify, path=(root, ckanext)) diff --git a/ckan/cli/generate.py b/ckan/cli/generate.py index dd30c4dca6a..e972cbaa8a6 100644 --- a/ckan/cli/generate.py +++ b/ckan/cli/generate.py @@ -1,20 +1,24 @@ # encoding: utf-8 +from __future__ import annotations -from __future__ import print_function +import contextlib import os +import json +import shutil +from typing import Optional import alembic.command import click from alembic.config import Config as AlembicConfig +from werkzeug.utils import import_string import ckan +from ckan import logic from ckan.cli.db import _resolve_alembic_config -import ckan.plugins.toolkit as tk -import uuid import string -import secrets from ckan.cli import error_shout +from ckan.common import config_declaration, config class CKANAlembicConfig(AlembicConfig): @@ -23,7 +27,7 @@ def get_template_directory(self): u"../contrib/alembic") -@click.group() +@click.group(short_help=u"Scaffolding for regular development tasks.") def generate(): """Scaffolding for regular development tasks. """ @@ -36,15 +40,15 @@ def generate(): help=u"Location to put the generated " u"template.", default=u'.') -def extension(output_dir): +def extension(output_dir: str): """Generate empty extension files to expand CKAN. """ try: from cookiecutter.main import cookiecutter except ImportError: - tk.error_shout(u"`cookiecutter` library is missing from import path.") - tk.error_shout(u"Make sure you have dev-dependencies installed:") - tk.error_shout(u"\tpip install -r dev-requirements.txt") + error_shout(u"`cookiecutter` library is missing from import path.") + error_shout(u"Make sure you have dev-dependencies installed:") + error_shout(u"\tpip install -r dev-requirements.txt") raise click.Abort() cur_loc = os.path.dirname(os.path.abspath(__file__)) @@ -58,8 +62,9 @@ def extension(output_dir): name = click.prompt(u"Extension's name", default=u"must begin 'ckanext-'") if not name.startswith(u"ckanext-"): - print(u"ERROR: Project name must start with 'ckanext-' > {}\n" - .format(name)) + error_shout( + u"ERROR: Project name must start with 'ckanext-' > {}\n" + .format(name)) else: break @@ -82,6 +87,8 @@ def extension(output_dir): project_short = name[8:].lower().replace(u'-', u'_') plugin_class_name = project_short.title().replace(u'_', u'') + u'Plugin' + include_examples = int(click.confirm( + "Do you want to include code examples?")) context = { u"project": name, u"description": description, @@ -91,7 +98,8 @@ def extension(output_dir): u"github_user_name": github, u"project_shortname": project_short, u"plugin_class_name": plugin_class_name, - u"_source": u"cli" + u"include_examples": include_examples, + u"_source": u"cli", } if output_dir == u'.': @@ -101,13 +109,43 @@ def extension(output_dir): cookiecutter(template_loc, no_input=True, extra_context=context, output_dir=output_dir) - print(u"\nWritten: {}/{}".format(output_dir, name)) + if not include_examples: + remove_code_examples( + os.path.join( + output_dir, context["project"], "ckanext", project_short)) + + click.echo(u"\nWritten: {}/{}".format(output_dir, name)) + + +_code_examples = [ + "cli.py", + "helpers.py", + "logic", + "views.py", + "tests/logic", + "tests/test_helpers.py", + "tests/test_views.py", +] + + +def remove_code_examples(root: str): + """Remove example files from extension's template. + """ + for item in _code_examples: + path = os.path.join(root, item) + with contextlib.suppress(FileNotFoundError): + if os.path.isdir(path): + shutil.rmtree(path) + else: + os.remove(path) @generate.command(name=u'config', short_help=u'Create a ckan.ini file.') @click.argument(u'output_path', nargs=1) -def make_config(output_path): +@click.option('-i', '--include-plugin', multiple=True, + help="Include config declaration from the given plugin") +def make_config(output_path: str, include_plugin: list[str]): u"""Generate a new CKAN configuration ini file.""" # Output to current directory if no path is specified @@ -117,18 +155,28 @@ def make_config(output_path): cur_loc = os.path.dirname(os.path.abspath(__file__)) template_loc = os.path.join(cur_loc, u'..', u'config', u'deployment.ini_tmpl') - template_variables = { - u'app_instance_uuid': uuid.uuid4(), - u'app_instance_secret': secrets.token_urlsafe(20)[:25] - } + config_declaration._reset() + config_declaration.load_core_declaration() + for plugin in include_plugin: + config_declaration.load_plugin(plugin) + + variables = { + "app_main_section": config_declaration.into_ini( + minimal=False, + include_docs=False, + ), + "default_section": config_declaration.into_ini( + minimal=False, + include_docs=True, + section="DEFAULT" + ) + } with open(template_loc, u'r') as file_in: template = string.Template(file_in.read()) - try: with open(output_path, u'w') as file_out: - file_out.writelines(template.substitute(template_variables)) - + file_out.writelines(template.substitute(variables)) except IOError as e: error_shout(e) raise click.Abort() @@ -143,24 +191,124 @@ def make_config(output_path): @click.option(u"-m", u"--message", help=u"Message string to use with `revision`.") -def migration(plugin, message): +def migration(plugin: str, message: str): """Create new alembic revision for DB migration. """ - import ckan.model - if not tk.config: - tk.error_shout(u'Config is not loaded') + if not config: + error_shout(u'Config is not loaded') raise click.Abort() - config = CKANAlembicConfig(_resolve_alembic_config(plugin)) - migration_dir = os.path.dirname(config.config_file_name) - config.set_main_option(u"sqlalchemy.url", - str(ckan.model.repo.metadata.bind.url)) - config.set_main_option(u'script_location', migration_dir) + alembic_config = CKANAlembicConfig(_resolve_alembic_config(plugin)) + assert alembic_config.config_file_name + migration_dir = os.path.dirname(alembic_config.config_file_name) + alembic_config.set_main_option("sqlalchemy.url", "") + alembic_config.set_main_option(u'script_location', migration_dir) if not os.path.exists(os.path.join(migration_dir, u'script.py.mako')): - alembic.command.init(config, migration_dir) + alembic.command.init(alembic_config, migration_dir) - rev = alembic.command.revision(config, message) + rev = alembic.command.revision(alembic_config, message) + rev_path = rev.path # type: ignore click.secho( - u"Revision file created. Now, you need to update it: \n\t{}".format( - rev.path), + f"Revision file created. Now, you need to update it: \n\t{rev_path}", fg=u"green") + + +_factories = { + "activity": "ckanext.activity.tests.conftest:ActivityFactory", + "api-token": "ckan.tests.factories:APIToken", + "dataset": "ckan.tests.factories:Dataset", + "group": "ckan.tests.factories:Group", + "organization": "ckan.tests.factories:Organization", + "resource": "ckan.tests.factories:Resource", + "resource-view": "ckan.tests.factories:ResourceView", + "user": "ckan.tests.factories:User", + "vocabulary": "ckan.tests.factories:Vocabulary", +} + + +@generate.command(context_settings={ + "allow_extra_args": True, "ignore_unknown_options": True +}) +@click.argument( + "category", required=False, type=click.Choice(list(_factories))) +@click.option( + "-f", "--factory-class", + help="Import path of the factory class that can generate an entity") +@click.option("-n", "--fake-count", type=int, default=1, + help="Number of entities to create") +@click.pass_context +def fake_data(ctx: click.Context, category: Optional[str], + factory_class: Optional[str], fake_count: int): + """Generate random entities of the given category. + + Either positional `category` or named `--factory-class`/`-f` argument must + be specified. `--factory-class` has higher priority, which means that + `category` is ignored if both arguments are provided at the same time. + + All the extra arguments that follows format `--NAME=VALUE` will be passed + into the entity factory. + + For instance: + + \b + ckan generate fake-data dataset + ckan generate fake-data dataset --title="My test dataset" + ckan generate fake-data dataset \\ + -f ckanext.myext.tests.factories.MyCustomDataset + + All the validation rules still apply. For example, if you have + `ckan.auth.create_unowned_dataset` config option set to `False`, + `--owner_org` must be supplied: + + \b + # use jq to obtain ID of the new organization + owner_org=$(ckan generate fake-data organization | jq .id -r) + ckan generate fake-data dataset --owner_org=$owner_org + + """ + try: + from ckan.tests.factories import CKANFactory + except ImportError as e: + error_shout(e) + error_shout("Make sure you have dev-dependencies installed:") + error_shout("\tpip install -r dev-requirements.txt") + raise click.Abort() + + if not factory_class: + if not category: + error_shout( + "Either `category` or `--factory-class` must be specified") + raise click.Abort() + factory_class = _factories[category] + if not factory_class: + error_shout("Either `category` or `factory_class` must be specified") + raise click.Abort() + + factory = import_string(factory_class, silent=True) + if not factory: + error_shout(f"{factory_class} cannot be imported") + raise click.Abort() + + if not issubclass(factory, CKANFactory): + error_shout("Factory must be a subclass of `{module}:{cls}`".format( + module=CKANFactory.__module__, + cls=CKANFactory.__name__, + )) + raise click.Abort() + + try: + extras = dict( + arg[2:].split("=") for arg in ctx.args if arg.startswith("--") + ) + except ValueError: + error_shout("Extra arguments must follow the format: --NAME=VALUE") + raise click.Abort() + + try: + for entity in factory.create_batch(fake_count, **extras): + # print entity as json, so that it can be stored in file or passed + # through jq-pipeline or similar tool + click.echo(json.dumps(entity)) + except logic.ValidationError as e: + error_shout(f"Cannot create entity: {e.error_dict}") + raise click.Abort() diff --git a/ckan/cli/jobs.py b/ckan/cli/jobs.py index cd9a4361af6..db35bec2b57 100644 --- a/ckan/cli/jobs.py +++ b/ckan/cli/jobs.py @@ -1,10 +1,10 @@ # encoding: utf-8 +from __future__ import annotations import click import ckan.lib.jobs as bg_jobs import ckan.logic as logic -import ckan.plugins as p from ckan.cli import error_shout @@ -16,7 +16,7 @@ def jobs(): @jobs.command(short_help=u"Start a worker.",) @click.option(u"--burst", is_flag=True, help=u"Start worker in burst mode.") @click.argument(u"queues", nargs=-1) -def worker(burst, queues): +def worker(burst: bool, queues: list[str]): """Start a worker that fetches jobs from queues and executes them. If no queue names are given then the worker listens to the default queue, this is equivalent to @@ -41,14 +41,14 @@ def worker(burst, queues): @jobs.command(name=u"list", short_help=u"List jobs.") @click.argument(u"queues", nargs=-1) -def list_jobs(queues): +def list_jobs(queues: list[str]): """List currently enqueued jobs from the given queues. If no queue names are given then the jobs from all queues are listed. """ data_dict = { u"queues": list(queues), } - jobs = p.toolkit.get_action(u"job_list")({u"ignore_auth": True}, data_dict) + jobs = logic.get_action(u"job_list")({u"ignore_auth": True}, data_dict) if not jobs: return click.secho(u"There are no pending jobs.", fg=u"green") for job in jobs: @@ -61,9 +61,9 @@ def list_jobs(queues): @jobs.command(short_help=u"Show details about a specific job.") @click.argument(u"id") -def show(id): +def show(id: str): try: - job = p.toolkit.get_action(u"job_show")( + job = logic.get_action(u"job_show")( {u"ignore_auth": True}, {u"id": id} ) except logic.NotFound: @@ -82,14 +82,14 @@ def show(id): @jobs.command(short_help=u"Cancel a specific job.") @click.argument(u"id") -def cancel(id): +def cancel(id: str): """Cancel a specific job. Jobs can only be canceled while they are enqueued. Once a worker has started executing a job it cannot be aborted anymore. """ try: - p.toolkit.get_action(u"job_cancel")( + logic.get_action(u"job_cancel")( {u"ignore_auth": True}, {u"id": id} ) except logic.NotFound: @@ -101,7 +101,7 @@ def cancel(id): @jobs.command(short_help=u"Cancel all jobs.") @click.argument(u"queues", nargs=-1) -def clear(queues): +def clear(queues: list[str]): """Cancel all jobs on the given queues. If no queue names are given then ALL queues are cleared. @@ -109,16 +109,16 @@ def clear(queues): data_dict = { u"queues": list(queues), } - queues = p.toolkit.get_action(u"job_clear")( + queues = logic.get_action(u"job_clear")( {u"ignore_auth": True}, data_dict ) - queues = (u'"{}"'.format(q) for q in queues) + queues = [u'"{}"'.format(q) for q in queues] click.secho(u"Cleared queue(s) {}".format(u", ".join(queues)), fg=u"green") @jobs.command(short_help=u"Enqueue a test job.") @click.argument(u"queues", nargs=-1) -def test(queues): +def test(queues: list[str]): """Enqueue a test job. If no queue names are given then the job is added to the default queue. If queue names are given then a separate test job is added to each of the queues. diff --git a/ckan/cli/less.py b/ckan/cli/less.py deleted file mode 100644 index d45dafaa178..00000000000 --- a/ckan/cli/less.py +++ /dev/null @@ -1,36 +0,0 @@ -# encoding: utf-8 - -import click -import subprocess -import os - -import six - -from ckan.common import config -from ckan.cli import error_shout - - -@click.command( - name=u'less', - short_help=u'Compile all root less documents into their CSS counterparts') -def less(): - command = (u'npm', u'run', u'build') - - public = config.get(u'ckan.base_public_folder') - - root = os.path.join(os.path.dirname(__file__), u'..', public, u'base') - root = os.path.abspath(root) - _compile_less(root, command, u'main') - - -def _compile_less(root, command, color): - click.echo(u'compile {}.css'.format(color)) - command = command + (u'--', u'--' + color) - - process = subprocess.Popen( - command, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - output = process.communicate() - for block in output: - click.echo(six.ensure_text(block)) diff --git a/ckan/cli/minify.py b/ckan/cli/minify.py deleted file mode 100644 index 82763ff7072..00000000000 --- a/ckan/cli/minify.py +++ /dev/null @@ -1,80 +0,0 @@ -# encoding: utf-8 - -import click -import os -import ckan.include.rjsmin as rjsmin -import ckan.include.rcssmin as rcssmin - -_exclude_dirs = [u'vendor'] - - -@click.command(name=u'minify') -@click.option( - u'--clean', is_flag=True, help=u'remove any minified files in the path.') -@click.argument(u'path', nargs=-1, type=click.Path()) -def minify(clean, path): - u'''Create minified versions of the given Javascript and CSS files.''' - for base_path in path: - if os.path.isfile(base_path): - if clean: - _clear_minifyed(base_path) - else: - _minify_file(base_path) - elif os.path.isdir(base_path): - for root, dirs, files in os.walk(base_path): - dirs[:] = [d for d in dirs if d not in _exclude_dirs] - for filename in files: - path = os.path.join(root, filename) - if clean: - _clear_minifyed(path) - else: - _minify_file(path) - else: - # Path is neither a file or a dir? - continue - - -def _clear_minifyed(path): - u'''Remove the minified version of the file''' - path_only, extension = os.path.splitext(path) - - if extension not in (u'.css', u'.js'): - # This is not a js or css file. - return - - if path_only.endswith(u'.min'): - click.echo(u'removing {}'.format(path)) - os.remove(path) - - -def _minify_file(path): - u'''Create the minified version of the given file. - - If the file is not a .js or .css file (e.g. it's a .min.js or .min.css - file, or it's some other type of file entirely) it will not be - minifed. - - :param path: The path to the .js or .css file to minify - - ''' - import ckan.lib.fanstatic_resources as fanstatic_resources - path_only, extension = os.path.splitext(path) - - if path_only.endswith(u'.min'): - # This is already a minified file. - return - - if extension not in (u'.css', u'.js'): - # This is not a js or css file. - return - - path_min = fanstatic_resources.min_path(path) - - source = open(path, u'r').read() - f = open(path_min, u'w') - if path.endswith(u'.css'): - f.write(rcssmin.cssmin(source)) - elif path.endswith(u'.js'): - f.write(rjsmin.jsmin(source)) - f.close() - click.echo(u"Minified file '{}'".format(path)) diff --git a/ckan/cli/notify.py b/ckan/cli/notify.py index 3e44da3b2aa..de68aa56aa6 100644 --- a/ckan/cli/notify.py +++ b/ckan/cli/notify.py @@ -1,23 +1,46 @@ # encoding: utf-8 +from logging import getLogger import click from ckan.model import Session, Package, DomainObjectOperation from ckan.model.modification import DomainObjectModificationExtension +from ckan.logic import NotAuthorized, ValidationError +from ckan.cli import error_shout +log = getLogger(__name__) -@click.group( - name=u'notify', - short_help=u'Send out modification notifications.' -) + +@click.group(name="notify", short_help="Send out modification notifications.") def notify(): pass -@notify.command( - name=u'replay', - short_help=u'Send out modification signals.' -) +@notify.command(name="replay", short_help="Send out modification signals.") def replay(): dome = DomainObjectModificationExtension() for package in Session.query(Package): dome.notify(package, DomainObjectOperation.changed) + + +@notify.command(name="send_emails", short_help="Send out Email notifications.") +def send_emails(): + """ Sends an email to users notifying about new activities. + + As currently implemented, it will only send notifications from dashboard + activity list if users have `activity_streams_email_notifications` set + in their profile. It will send emails with updates depending + on the `ckan.email_notifications_since` config. (default: 2 days.) + """ + import ckan.logic as logic + import ckan.lib.mailer as mailer + from ckan.types import Context + from typing import cast + + site_user = logic.get_action("get_site_user")({"ignore_auth": True}, {}) + context = cast(Context, {"user": site_user["name"]}) + try: + logic.get_action("send_email_notifications")(context, {}) + except (NotAuthorized, ValidationError, mailer.MailerException) as e: + error_shout(e) + except KeyError: + error_shout("`activity` plugin is not enabled") diff --git a/ckan/cli/plugin_info.py b/ckan/cli/plugin_info.py index 9f6a8ac47d2..ac27aaeebf0 100644 --- a/ckan/cli/plugin_info.py +++ b/ckan/cli/plugin_info.py @@ -1,6 +1,10 @@ # encoding: utf-8 +from __future__ import annotations + +from typing import Any, Callable import click +import ckan.plugins as p @click.command( @@ -50,19 +54,19 @@ def plugin_info(): click.echo() -def _template_helpers(plugin_class): +def _template_helpers(plugin_class: p.ITemplateHelpers): u''' Return readable helper function info. ''' helpers = plugin_class.get_helpers() return _function_info(helpers) -def _actions(plugin_class): +def _actions(plugin_class: p.IActions): u''' Return readable action function info. ''' actions = plugin_class.get_actions() return _function_info(actions) -def _function_info(functions): +def _function_info(functions: dict[str, Callable[..., Any]]): u''' Take a dict of functions and output readable info ''' import inspect output = [] diff --git a/ckan/cli/profile.py b/ckan/cli/profile.py index d88613afc0d..928d42cfd6a 100644 --- a/ckan/cli/profile.py +++ b/ckan/cli/profile.py @@ -4,15 +4,14 @@ import traceback import click - -from ckan.cli import error_shout +from . import error_shout @click.group( short_help=u"Code speed profiler.", invoke_without_command=True, ) @click.pass_context -def profile(ctx): +def profile(ctx: click.Context): """Provide a ckan url and it will make the request and record how long each function call took in a file that can be read by pstats.Stats (command-line) or runsnakerun (gui). @@ -30,21 +29,21 @@ def profile(ctx): """ if ctx.invoked_subcommand is None: - ctx.invoke(profile) + ctx.invoke(main) -@profile.command(short_help=u"Code speed profiler.",) +@profile.command('profile', short_help=u"Code speed profiler.",) @click.argument(u"url") @click.argument(u"user", required=False, default=u"visitor") -def profile(url, user): +def main(url: str, user: str): import cProfile from ckan.tests.helpers import _get_test_app app = _get_test_app() - def profile_url(url): + def profile_url(url: str): # type: ignore # noqa try: - res = app.get( + app.get( url, status=[200], extra_environ={u"REMOTE_USER": str(user)} ) except KeyboardInterrupt: diff --git a/ckan/cli/sass.py b/ckan/cli/sass.py new file mode 100644 index 00000000000..1cb3afda1f8 --- /dev/null +++ b/ckan/cli/sass.py @@ -0,0 +1,47 @@ +# encoding: utf-8 +from __future__ import annotations + +import subprocess +import os + +import click +import six + +from ckan.common import config + + +@click.command( + name='sass', + short_help='Compile all root sass documents into their CSS counterparts') +@click.option( + '-d', + '--debug', + is_flag=True, + help="Compile css with sourcemaps.") +def sass(debug: bool): + command = ('npm', 'run', 'build') + + public = config.get('ckan.base_public_folder') + + root = os.path.join(os.path.dirname(__file__), '..', public, 'base') + root = os.path.abspath(root) + _compile_sass(root, command, 'main', debug) + + +def _compile_sass( + root: str, + command: tuple[str, ...], + color: str, + debug: bool): + click.echo('compile {}.css'.format(color)) + command = command + ('--', '--' + color) + if debug: + command = command + ('--debug',) + + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output = process.communicate() + for block in output: + click.echo(six.ensure_text(block)) diff --git a/ckan/cli/search_index.py b/ckan/cli/search_index.py index 3b4bedc6d2a..bb1304081fa 100644 --- a/ckan/cli/search_index.py +++ b/ckan/cli/search_index.py @@ -1,11 +1,15 @@ # encoding: utf-8 +from __future__ import annotations import multiprocessing as mp import click import sqlalchemy as sa - -import ckan.plugins.toolkit as tk +from ckan.common import config +from ckan.lib.search import query_for +import ckan.logic as logic +import ckan.model as model +from . import error_shout @click.group(name=u'search-index', short_help=u'Search index commands') @@ -18,7 +22,6 @@ def search_index(): @click.option(u'-v', u'--verbose', is_flag=True) @click.option(u'-i', u'--force', is_flag=True, help=u'Ignore exceptions when rebuilding the index') -@click.option(u'-r', u'--refresh', help=u'Refresh current index', is_flag=True) @click.option(u'-o', u'--only-missing', help=u'Index non indexed datasets only', is_flag=True) @click.option(u'-q', u'--quiet', help=u'Do not output index rebuild progress', @@ -28,9 +31,12 @@ def search_index(): u'ensures that changes are immediately available on the' u'search, but slows significantly the process. Default' u'is false.') +@click.option('-c', '--clear', help='Clear the index before reindexing', + is_flag=True) @click.argument(u'package_id', required=False) def rebuild( - verbose, force, refresh, only_missing, quiet, commit_each, package_id + verbose: bool, force: bool, only_missing: bool, quiet: bool, + commit_each: bool, package_id: str, clear: bool ): u''' Rebuild search index ''' from ckan.lib.search import rebuild, commit @@ -39,11 +45,11 @@ def rebuild( rebuild(package_id, only_missing=only_missing, force=force, - refresh=refresh, defer_commit=(not commit_each), - quiet=quiet) + quiet=quiet and not verbose, + clear=clear) except Exception as e: - tk.error_shout(e) + error_shout(e) if not commit_each: commit() @@ -56,7 +62,7 @@ def check(): @search_index.command(name=u'show', short_help=u'Show index of a dataset') @click.argument(u'dataset_name') -def show(dataset_name): +def show(dataset_name: str): from ckan.lib.search import show index = show(dataset_name) @@ -65,7 +71,7 @@ def show(dataset_name): @search_index.command(name=u'clear', short_help=u'Clear the search index') @click.argument(u'dataset_name', required=False) -def clear(dataset_name): +def clear(dataset_name: str): from ckan.lib.search import clear, clear_all if dataset_name: @@ -74,28 +80,101 @@ def clear(dataset_name): clear_all() +def get_orphans() -> list[str]: + search = None + indexed_package_ids = [] + while search is None or len(indexed_package_ids) < search['count']: + search = logic.get_action('package_search')({}, { + 'q': '*:*', + 'fl': 'id', + 'start': len(indexed_package_ids), + 'rows': 1000}) + indexed_package_ids += search['results'] + + package_ids = {r[0] for r in model.Session.query(model.Package.id)} + + orphaned_package_ids = [] + + for indexed_package_id in indexed_package_ids: + if indexed_package_id['id'] not in package_ids: + orphaned_package_ids.append(indexed_package_id['id']) + + return orphaned_package_ids + + +@search_index.command( + name=u'list-orphans', + short_help=u'Lists any non-existant packages in the search index' +) +def list_orphans_command(): + orphaned_package_ids = get_orphans() + if len(orphaned_package_ids): + click.echo(orphaned_package_ids) + click.echo("Found {} orphaned package(s).".format( + len(orphaned_package_ids) + )) + + +@search_index.command( + name=u'clear-orphans', + short_help=u'Clear any non-existant packages in the search index' +) +@click.option(u'-v', u'--verbose', is_flag=True) +def clear_orphans(verbose: bool = False): + for orphaned_package_id in get_orphans(): + if verbose: + click.echo("Clearing search index for dataset {}...".format( + orphaned_package_id + )) + clear(orphaned_package_id) + + +@search_index.command( + name=u'list-unindexed', + short_help=u'Lists any missing packages from the search index' +) +def list_unindexed(): + packages = model.Session.query(model.Package.id) + if config.get('ckan.search.remove_deleted_packages'): + packages = packages.filter(model.Package.state != 'deleted') + + package_ids = [r[0] for r in packages.all()] + + package_query = query_for(model.Package) + indexed_pkg_ids = set(package_query.get_all_entity_ids( + max_results=len(package_ids))) + # Packages not indexed + unindexed_package_ids = set(package_ids) - indexed_pkg_ids + + if len(unindexed_package_ids): + click.echo(unindexed_package_ids) + click.echo("Found {} unindexed package(s).".format( + len(unindexed_package_ids) + )) + + @search_index.command(name=u'rebuild-fast', short_help=u'Reindex with multiprocessing') def rebuild_fast(): from ckan.lib.search import commit - db_url = tk.config['sqlalchemy.url'] + db_url = config['sqlalchemy.url'] engine = sa.create_engine(db_url) package_ids = [] result = engine.execute(u"select id from package where state = 'active';") for row in result: package_ids.append(row[0]) - def start(ids): + def start(ids: list[str]): from ckan.lib.search import rebuild rebuild(package_ids=ids) - def chunks(l, n): - u""" Yield n successive chunks from l.""" - newn = int(len(l) / n) + def chunks(list_: list[str], n: int): + u""" Yield n successive chunks from list_""" + newn = int(len(list_) / n) for i in range(0, n - 1): - yield l[i * newn:i * newn + newn] - yield l[n * newn - newn:] + yield list_[i * newn:i * newn + newn] + yield list_[n * newn - newn:] processes = [] @@ -110,4 +189,4 @@ def chunks(l, n): process.join() commit() except Exception as e: - click.echo(e.message) + error_shout(e) diff --git a/ckan/cli/seed.py b/ckan/cli/seed.py deleted file mode 100644 index 764c916957f..00000000000 --- a/ckan/cli/seed.py +++ /dev/null @@ -1,85 +0,0 @@ -# encoding: utf-8 - -import logging - -import click - -from ckan.lib.create_test_data import CreateTestData - -log = logging.getLogger(__name__) - - -@click.group(short_help=u'Create test data in the database.') -def seed(): - u'''Create test data in the database. - - Tests can also delete the created objects easily with the delete() method. - ''' - pass - - -@seed.command(short_help=u'Annakarenina and warandpeace.') -@click.pass_context -def basic(ctx): - flask_app = ctx.meta['flask_app'] - with flask_app.test_request_context(): - CreateTestData.create_basic_test_data() - - -@seed.command(short_help=u'Realistic data to test search.') -@click.pass_context -def search(ctx): - flask_app = ctx.meta['flask_app'] - with flask_app.test_request_context(): - CreateTestData.create_search_test_data() - - -@seed.command(short_help=u'Government style data.') -@click.pass_context -def gov(ctx): - flask_app = ctx.meta['flask_app'] - with flask_app.test_request_context(): - CreateTestData.create_gov_test_data() - - -@seed.command(short_help=u'Package relationships data.') -@click.pass_context -def family(ctx): - flask_app = ctx.meta['flask_app'] - with flask_app.test_request_context(): - CreateTestData.create_family_test_data() - - -@seed.command(short_help=u'Create a user "tester" with api key "tester".') -@click.pass_context -def user(ctx): - flask_app = ctx.meta['flask_app'] - with flask_app.test_request_context(): - CreateTestData.create_test_user() - click.echo( - u'Created user {0} with password {0} and apikey {0}'.format(u'tester') - ) - - -@seed.command(short_help=u'Test translations of terms.') -@click.pass_context -def translations(ctx): - flask_app = ctx.meta['flask_app'] - with flask_app.test_request_context(): - CreateTestData.create_translations_test_data() - - -@seed.command(short_help=u'Some test vocabularies.') -@click.pass_context -def vocabs(ctx): - flask_app = ctx.meta['flask_app'] - with flask_app.test_request_context(): - CreateTestData.create_vocabs_test_data() - - -@seed.command(short_help=u'Hierarchy of groups.') -@click.pass_context -def hierarchy(ctx): - flask_app = ctx.meta['flask_app'] - with flask_app.test_request_context(): - CreateTestData.create_group_hierarchy_test_data() diff --git a/ckan/cli/server.py b/ckan/cli/server.py index 1be01d49241..5d1f69252cf 100644 --- a/ckan/cli/server.py +++ b/ckan/cli/server.py @@ -1,58 +1,126 @@ # encoding: utf-8 +from __future__ import annotations +from ckan.exceptions import CkanDeprecationWarning import logging +import warnings +from typing import Optional import click from werkzeug.serving import run_simple +from werkzeug.middleware.dispatcher import DispatcherMiddleware -import ckan.plugins.toolkit as tk from ckan.common import config +from . import error_shout log = logging.getLogger(__name__) +DEFAULT_HOST = "localhost" +DEFAULT_PORT = 5000 + @click.command(u"run", short_help=u"Start development server") -@click.option(u"-H", u"--host", default=u"localhost", help=u"Set host") -@click.option(u"-p", u"--port", default=5000, help=u"Set port") +@click.option(u"-H", u"--host", help=u"Host name") +@click.option(u"-p", u"--port", help=u"Port number") @click.option(u"-r", u"--disable-reloader", is_flag=True, help=u"Disable reloader") +@click.option(u"-E", u"--passthrough-errors", is_flag=True, + help=u"Disable error caching (useful to hook debuggers)") @click.option( u"-t", u"--threaded", is_flag=True, help=u"Handle each request in a separate thread" ) -@click.option(u"-e", u"--extra-files", multiple=True) @click.option( u"--processes", type=int, default=0, help=u"Maximum number of concurrent processes" ) +@click.option( + u"-e", u"--extra-files", multiple=True, + help=u"Additional files that should be watched for server reloading" + " (you can provide multiple values)") +@click.option( + u"-C", u"--ssl-cert", default=None, + help=u"Certificate file to use to enable SSL. Passing 'adhoc' will " + " automatically generate a new one (on each server reload).") +@click.option( + u"-K", u"--ssl-key", default=None, + help=u"Key file to use to enable SSL. Passing 'adhoc' will " + " automatically generate a new one (on each server reload).") +@click.option( + u"-P", u"--prefix", default="", + help=u"Run ckan in prefix path." +) @click.pass_context -def run(ctx, host, port, disable_reloader, threaded, extra_files, processes): +def run(ctx: click.Context, host: str, port: str, disable_reloader: bool, + passthrough_errors: bool, threaded: bool, extra_files: list[str], + processes: int, ssl_cert: Optional[str], ssl_key: Optional[str], + prefix: Optional[str]): u"""Runs the Werkzeug development server""" - use_reloader = not disable_reloader - threaded = threaded or tk.asbool(config.get(u"ckan.devserver.threaded")) - processes = processes or tk.asint( - config.get(u"ckan.devserver.multiprocess", 1) - ) - if threaded and processes > 1: - tk.error_shout(u"Cannot have a multithreaded and multi process server") - raise click.Abort() - log.info(u"Running server {0} on port {1}".format(host, port)) + if config.get("debug"): + warnings.filterwarnings("default", category=CkanDeprecationWarning) - config_extra_files = tk.aslist( - config.get(u"ckan.devserver.watch_patterns") - ) + # passthrough_errors overrides conflicting options + if passthrough_errors: + disable_reloader = True + threaded = False + processes = 1 + + # Reloading + use_reloader = not disable_reloader + config_extra_files = config.get(u"ckan.devserver.watch_patterns") extra_files = list(extra_files) + [ config[u"__file__"] ] + config_extra_files + # Threads and processes + threaded = threaded or config.get(u"ckan.devserver.threaded") + processes = processes or config.get(u"ckan.devserver.multiprocess") + if threaded and processes > 1: + error_shout(u"Cannot have a multithreaded and multi process server") + raise click.Abort() + + # SSL + cert_file = ssl_cert or config.get('ckan.devserver.ssl_cert') + key_file = ssl_key or config.get('ckan.devserver.ssl_key') + + if cert_file and key_file: + if cert_file == key_file == 'adhoc': + ssl_context = 'adhoc' + else: + ssl_context = (ssl_cert, ssl_key) + else: + ssl_context = None + + if prefix: + if not prefix.startswith(u'/'): + error_shout(u"Prefix must start with /, example /data.") + raise click.Abort() + ctx.obj.app = DispatcherMiddleware(ctx.obj.app, { + prefix: ctx.obj.app + }) + + host = host or config.get('ckan.devserver.host') + port = port or config.get('ckan.devserver.port') + try: + port_int = int(port) + except ValueError: + error_shout(u"Server port must be an integer, not {}".format(port)) + raise click.Abort() + + log.info(u"Running CKAN on {scheme}://{host}:{port}{prefix}".format( + scheme='https' if ssl_context else 'http', host=host, port=port_int, + prefix=prefix)) + run_simple( host, - port, + port_int, ctx.obj.app, use_reloader=use_reloader, use_evalex=True, threaded=threaded, processes=processes, extra_files=extra_files, + ssl_context=ssl_context, + passthrough_errors=passthrough_errors, ) diff --git a/ckan/cli/shell.py b/ckan/cli/shell.py new file mode 100644 index 00000000000..7fffef529ca --- /dev/null +++ b/ckan/cli/shell.py @@ -0,0 +1,63 @@ +# encoding: utf-8 +import click +import logging + +import ckan.model as model + +from typing import Any, Mapping + +from ckan.plugins import toolkit + + +log = logging.getLogger(__name__) + + +_banner = """ +****** Welcome to the CKAN shell ****** + +This session has some variables pre-populated: + - app (CKAN Application object) + - config (CKAN config dictionary) + - model (CKAN model module to access the Database) + - toolkit (CKAN toolkit module) + """ + + +def ipython(namespace: Mapping[str, Any], banner: str) -> None: + import IPython + from traitlets.config.loader import Config + + c = Config() + c.TerminalInteractiveShell.banner2 = banner # type: ignore + + IPython.start_ipython([], user_ns=namespace, config=c) + + +def python(namespace: Mapping[str, Any], banner: str) -> None: + import code + code.interact(banner=banner, local=namespace) + + +@click.command() +@click.help_option("-h", "--help") +@click.pass_context +def shell(ctx: click.Context): + """Run an interactive IPython shell with the context of the + CKAN instance. + + It will try to use IPython, if not installed it will callback + to the default Python's shell. + """ + + namespace = { + "app": ctx.obj.app._wsgi_app, + "model": model, + "config": ctx.obj.config, + "toolkit": toolkit, + } + + try: + ipython(namespace, _banner) + except ImportError: + log.debug("`ipython` library is missing. Using default python shell.") + python(namespace, _banner) diff --git a/ckan/cli/sysadmin.py b/ckan/cli/sysadmin.py index b312b303a99..00227f7d9f3 100644 --- a/ckan/cli/sysadmin.py +++ b/ckan/cli/sysadmin.py @@ -1,7 +1,7 @@ # encoding: utf-8 +from __future__ import annotations import click -from six import text_type import ckan.model as model from ckan.cli import error_shout @@ -13,7 +13,7 @@ invoke_without_command=True, ) @click.pass_context -def sysadmin(ctx): +def sysadmin(ctx: click.Context): """Gives sysadmin rights to a named user. """ @@ -44,16 +44,16 @@ def list_sysadmins(): @click.argument(u"username") @click.argument(u"args", nargs=-1) @click.pass_context -def add(ctx, username, args): - user = model.User.by_name(text_type(username)) +def add(ctx: click.Context, username: str, args: list[str]): + user = model.User.by_name(str(username)) if not user: click.secho(u'User "%s" not found' % username, fg=u"red") if click.confirm( u"Create new user: %s?" % username, default=True, abort=True ): ctx.forward(add_user) - user = model.User.by_name(text_type(username)) - + user = model.User.by_name(str(username)) + assert user user.sysadmin = True model.Session.add(user) model.repo.commit_and_remove() @@ -62,8 +62,8 @@ def add(ctx, username, args): @sysadmin.command(help=u"Removes user from sysadmins.") @click.argument(u"username") -def remove(username): - user = model.User.by_name(text_type(username)) +def remove(username: str): + user = model.User.by_name(str(username)) if not user: return error_shout(u'Error: user "%s" not found!' % username) user.sysadmin = False diff --git a/ckan/cli/tracking.py b/ckan/cli/tracking.py index f7a6c13000b..281df435369 100644 --- a/ckan/cli/tracking.py +++ b/ckan/cli/tracking.py @@ -1,14 +1,21 @@ # encoding: utf-8 -import ckan.model as model -import click import datetime import csv + +from typing import NamedTuple, Optional + +import click + +import ckan.model as model import ckan.logic as logic -from collections import namedtuple from ckan.cli import error_shout -_ViewCount = namedtuple(u'ViewCount', u'id name count') + +class ViewCount(NamedTuple): + id: str + name: str + count: int @click.group(name=u'tracking', short_help=u'Update tracking statistics') @@ -18,24 +25,26 @@ def tracking(): @tracking.command() @click.argument(u'start_date', required=False) -def update(start_date): +def update(start_date: Optional[str]): engine = model.meta.engine + assert engine update_all(engine, start_date) @tracking.command() @click.argument(u'output_file', type=click.Path()) @click.argument(u'start_date', required=False) -def export(output_file, start_date): +def export(output_file: str, start_date: Optional[str]): engine = model.meta.engine + assert engine update_all(engine, start_date) export_tracking(engine, output_file) -def update_all(engine, start_date=None): +def update_all(engine: model.Engine, start_date: Optional[str] = None): if start_date: - start_date = datetime.datetime.strptime(start_date, u'%Y-%m-%d') + date = datetime.datetime.strptime(start_date, u'%Y-%m-%d') else: # No date given. See when we last have data for and get data # from 2 days before then in case new data is available. @@ -44,26 +53,26 @@ def update_all(engine, start_date=None): ORDER BY tracking_date DESC LIMIT 1;''' result = engine.execute(sql).fetchall() if result: - start_date = result[0][u'tracking_date'] - start_date += datetime.timedelta(-2) + date = result[0][u'tracking_date'] + date += datetime.timedelta(-2) # convert date to datetime combine = datetime.datetime.combine - start_date = combine(start_date, datetime.time(0)) + date = combine(date, datetime.time(0)) else: - start_date = datetime.datetime(2011, 1, 1) - start_date_solrsync = start_date + date = datetime.datetime(2011, 1, 1) + start_date_solrsync = date end_date = datetime.datetime.now() - while start_date < end_date: - stop_date = start_date + datetime.timedelta(1) - update_tracking(engine, start_date) - click.echo(u'tracking updated for {}'.format(start_date)) - start_date = stop_date + while date < end_date: + stop_date = date + datetime.timedelta(1) + update_tracking(engine, date) + click.echo(u'tracking updated for {}'.format(date)) + date = stop_date update_tracking_solr(engine, start_date_solrsync) -def _total_views(engine): +def _total_views(engine: model.Engine): sql = u''' SELECT p.id, p.name, @@ -73,10 +82,10 @@ def _total_views(engine): GROUP BY p.id, p.name ORDER BY total_views DESC ''' - return [_ViewCount(*t) for t in engine.execute(sql).fetchall()] + return [ViewCount(*t) for t in engine.execute(sql).fetchall()] -def _recent_views(engine, measure_from): +def _recent_views(engine: model.Engine, measure_from: datetime.date): sql = u''' SELECT p.id, p.name, @@ -88,15 +97,15 @@ def _recent_views(engine, measure_from): ORDER BY total_views DESC ''' return [ - _ViewCount(*t) for t in engine.execute( + ViewCount(*t) for t in engine.execute( sql, measure_from=str(measure_from) ).fetchall() ] -def export_tracking(engine, output_filename): +def export_tracking(engine: model.Engine, output_filename: str): u'''Write tracking summary to a csv file.''' - HEADINGS = [ + headings = [ u'dataset id', u'dataset name', u'total views', @@ -109,7 +118,7 @@ def export_tracking(engine, output_filename): with open(output_filename, u'w') as fh: f_out = csv.writer(fh) - f_out.writerow(HEADINGS) + f_out.writerow(headings) recent_views_for_id = dict((r.id, r.count) for r in recent_views) f_out.writerows([(r.id, r.name, @@ -118,8 +127,8 @@ def export_tracking(engine, output_filename): for r in total_views]) -def update_tracking(engine, summary_date): - PACKAGE_URL = u'/dataset/' +def update_tracking(engine: model.Engine, summary_date: datetime.datetime): + package_url = u'/dataset/' # clear out existing data before adding new sql = u'''DELETE FROM tracking_summary WHERE tracking_date='%s'; ''' % summary_date @@ -150,7 +159,7 @@ def update_tracking(engine, summary_date): ,'~~not~found~~') WHERE t.package_id IS NULL AND tracking_type = 'page';''' - engine.execute(sql, PACKAGE_URL) + engine.execute(sql, package_url) # update summary totals for resources sql = u'''UPDATE tracking_summary t1 @@ -191,13 +200,13 @@ def update_tracking(engine, summary_date): engine.execute(sql) -def update_tracking_solr(engine, start_date): +def update_tracking_solr(engine: model.Engine, start_date: datetime.datetime): sql = u'''SELECT package_id FROM tracking_summary where package_id!='~~not~found~~' and tracking_date >= %s;''' results = engine.execute(sql, start_date) - package_ids = set() + package_ids: set[str] = set() for row in results: package_ids.add(row[u'package_id']) diff --git a/ckan/cli/translation.py b/ckan/cli/translation.py index 972c11fa91a..05fb0d44e9e 100644 --- a/ckan/cli/translation.py +++ b/ckan/cli/translation.py @@ -1,12 +1,14 @@ # encoding: utf-8 +from __future__ import annotations import polib import re import logging import os +from typing import Any, cast + import click -import six from ckan.common import config from ckan.lib.i18n import build_js_translations @@ -54,7 +56,7 @@ def mangle(): u'|\\%((\\d)*\\$)?' + spf_reg_ex + u')' for entry in po: - msg = entry.msgid.encode(u'utf-8') + msg = entry.msgid matches = re.finditer(extract_reg_ex, msg) length = len(msg) position = 0 @@ -82,10 +84,11 @@ def mangle(): u'check-po', short_help=u'Check po files for common mistakes' ) @click.argument(u'files', nargs=-1, type=click.Path(exists=True)) -def check_po(files): +def check_po(files: list[str]): for file in files: errors = check_po_file(file) for msgid, msgstr in errors: + click.echo(file) click.echo(u"Format specifiers don't match:") click.echo( u'\t{} -> {}'.format( @@ -99,7 +102,7 @@ def check_po(files): 'with the ones on the pot file' ) @click.argument(u'files', nargs=-1, type=click.Path(exists=True)) -def sync_po_msgids(files): +def sync_po_msgids(files: list[str]): i18n_path = get_i18n_path() pot_path = os.path.join(i18n_path, u'ckan.pot') po = polib.pofile(pot_path) @@ -111,11 +114,11 @@ def sync_po_msgids(files): sync_po_file_msgids(entries_to_change, path) -def normalize_string(s): +def normalize_string(s: str): return re.sub(r'\s\s+', ' ', s).strip() -def sync_po_file_msgids(entries_to_change, path): +def sync_po_file_msgids(entries_to_change: dict[str, Any], path: str): po = polib.pofile(path) cnt = 0 @@ -134,11 +137,12 @@ def sync_po_file_msgids(entries_to_change, path): ) -def get_i18n_path(): - return config.get(u'ckan.i18n_directory', os.path.join(ckan_path, u'i18n')) +def get_i18n_path() -> str: + return config.get( + u'ckan.i18n_directory') or os.path.join(ckan_path, u'i18n') -def simple_conv_specs(s): +def simple_conv_specs(s: str): '''Return the simple Python string conversion specifiers in the string s. e.g. ['%s', '%i'] @@ -149,7 +153,7 @@ def simple_conv_specs(s): return simple_conv_specs_re.findall(s) -def mapping_keys(s): +def mapping_keys(s: str): '''Return a sorted list of the mapping keys in the string s. e.g. ['%(name)s', '%(age)i'] @@ -160,7 +164,7 @@ def mapping_keys(s): return sorted(mapping_keys_re.findall(s)) -def replacement_fields(s): +def replacement_fields(s: str): '''Return a sorted list of the Python replacement fields in the string s. e.g. ['{}', '{2}', '{object}', '{target}'] @@ -171,28 +175,31 @@ def replacement_fields(s): return sorted(repl_fields_re.findall(s)) -def check_translation(validator, msgid, msgstr): +def check_translation(validator: Any, msgid: str, msgstr: str): if not validator(msgid) == validator(msgstr): return msgid, msgstr -def check_po_file(path): - errors = [] +def check_po_file(path: str): + errors: list[tuple[str, str]] = [] po = polib.pofile(path) for entry in po.translated_entries(): if entry.msgid_plural and entry.msgstr_plural: for function in ( simple_conv_specs, mapping_keys, replacement_fields ): - for key, msgstr in six.iteritems(entry.msgstr_plural): + # typechecker thinks it's a list of strings + plurals = cast("dict[str, str]", entry.msgstr_plural) + for key in plurals.keys(): if key == u'0': error = check_translation( - function, entry.msgid, entry.msgstr_plural[key] + function, entry.msgid, + plurals[key] ) else: error = check_translation( function, entry.msgid_plural, - entry.msgstr_plural[key] + plurals[key] ) if error: errors.append(error) diff --git a/ckan/cli/user.py b/ckan/cli/user.py index 08c715b906f..ecc7d73c15e 100644 --- a/ckan/cli/user.py +++ b/ckan/cli/user.py @@ -1,16 +1,18 @@ # encoding: utf-8 +from __future__ import annotations import logging -import sys -from pprint import pprint +from typing import cast import six import click -from six import text_type import ckan.logic as logic -import ckan.plugins as plugin +import ckan.model as model from ckan.cli import error_shout +from ckan.common import json +from ckan.types import Context +from ckan.lib.helpers import helper_functions as h log = logging.getLogger(__name__) @@ -25,7 +27,7 @@ def user(): @click.argument(u'username') @click.argument(u'args', nargs=-1) @click.pass_context -def add_user(ctx, username, args): +def add_user(ctx: click.Context, username: str, args: list[str]): u'''Add new user if we use ckan sysadmin add or ckan user add ''' @@ -52,22 +54,21 @@ def add_user(ctx, username, args): if u'fullname' in data_dict: data_dict['fullname'] = six.ensure_text(data_dict['fullname']) - # pprint(u'Creating user: %r' % username) + import ckan.logic as logic + import ckan.model as model try: - import ckan.logic as logic - import ckan.model as model - site_user = logic.get_action(u'get_site_user')({ + site_user = logic.get_action(u'get_site_user')(cast(Context, { u'model': model, - u'ignore_auth': True}, + u'ignore_auth': True}), {} ) - context = { + context = cast(Context, { u'model': model, u'session': model.Session, u'ignore_auth': True, u'user': site_user['name'], - } + }) flask_app = ctx.meta['flask_app'] # Current user is tested agains sysadmin role during model # dictization, thus we need request context @@ -80,7 +81,7 @@ def add_user(ctx, username, args): raise click.Abort() -def get_user_str(user): +def get_user_str(user: model.User): user_str = u'name=%s' % user.name if user.name != user.display_name: user_str += u' display=%s' % user.display_name @@ -100,33 +101,32 @@ def list_users(): @user.command(u'remove', short_help=u'Remove user') @click.argument(u'username') @click.pass_context -def remove_user(ctx, username): - import ckan.model as model +def remove_user(ctx: click.Context, username: str): if not username: error_shout(u'Please specify the username to be removed') return site_user = logic.get_action(u'get_site_user')({u'ignore_auth': True}, {}) - context = {u'user': site_user[u'name']} + context: Context = {u'user': site_user[u'name']} with ctx.meta['flask_app'].test_request_context(): - plugin.toolkit.get_action(u'user_delete')(context, {u'id': username}) + logic.get_action(u'user_delete')(context, {u'id': username}) click.secho(u'Deleted user: %s' % username, fg=u'green', bold=True) @user.command(u'show', short_help=u'Show user') @click.argument(u'username') -def show_user(username): +def show_user(username: str): import ckan.model as model if not username: error_shout(u'Please specify the username for the user') return - user = model.User.get(text_type(username)) + user = model.User.get(str(username)) click.secho(u'User: %s' % user) @user.command(u'setpass', short_help=u'Set password for the user') @click.argument(u'username') -def set_password(username): +def set_password(username: str): import ckan.model as model if not username: error_shout(u'Need name of the user.') @@ -142,3 +142,110 @@ def set_password(username): user.password = password model.repo.commit_and_remove() click.secho(u'Password updated!', fg=u'green', bold=True) + + +@user.group() +def token(): + """Manage API Tokens""" + pass + + +@token.command(u"add", context_settings=dict(ignore_unknown_options=True)) +@click.argument(u"username") +@click.argument(u"token_name") +@click.argument(u"extras", type=click.UNPROCESSED, nargs=-1) +@click.option( + u"--json", + "json_str", + metavar=u"EXTRAS", + default=u"{}", + help=u"Valid JSON object with additional fields for api_token_create", +) +@click.option( + "--quiet", + "-q", + is_flag=True, + help="Output just the token itself (useful in automated scripts)", +) +def add_token( + username: str, token_name: str, extras: list[str], json_str: str, + quiet: bool): + """Create a new API Token for the given user. + + Arbitrary fields can be passed in the form `key=value` or using + the --json option, containing a JSON encoded object. When both provided, + `key=value` fields will take precedence and will replace the + corresponding keys from the --json object. + + Example: + + ckan user token add john_doe new_token x=y --json '{"prop": "value"}' + + """ + data_dict = json.loads(json_str) + for chunk in extras: + try: + key, value = chunk.split(u"=") + except ValueError: + error_shout( + u"Extras must be passed as `key=value`. Got: {}".format( + chunk + ) + ) + raise click.Abort() + data_dict[key] = value + + data_dict.update({u"user": username, u"name": token_name}) + try: + token = logic.get_action(u"api_token_create")( + {u"ignore_auth": True}, data_dict + ) + except logic.NotFound as e: + error_shout(e) + raise click.Abort() + if not quiet: + click.secho(u"API Token created:", fg=u"green") + click.echo(u"\t", nl=False) + click.echo(token[u"token"]) + + +@token.command(u"revoke") +@click.argument(u"id") +def revoke_token(id: str): + """Remove API Token with the given ID""" + if not model.ApiToken.revoke(id): + error_shout(u"API Token not found") + raise click.Abort() + click.secho(u"API Token has been revoked", fg=u"green") + + +@token.command(u"list") +@click.argument(u"username") +def list_tokens(username: str): + """List all API Tokens for the given user""" + try: + tokens = logic.get_action(u"api_token_list")( + {u"ignore_auth": True}, {u"user": username} + ) + except logic.NotFound as e: + error_shout(e) + raise click.Abort() + if not tokens: + click.secho(u"No tokens have been created for user yet", fg=u"red") + return + click.echo(u"Tokens([id] name - lastAccess):") + + for token in tokens: + last_access = token[u"last_access"] + if last_access: + accessed = h.date_str_to_datetime( + last_access + ).isoformat(u" ", u"seconds") + + else: + accessed = u"Never" + click.echo( + u"\t[{id}] {name} - {accessed}".format( + name=token[u"name"], id=token[u"id"], accessed=accessed + ) + ) diff --git a/ckan/cli/views.py b/ckan/cli/views.py index cf65047efe3..760e2fc7c12 100644 --- a/ckan/cli/views.py +++ b/ckan/cli/views.py @@ -1,6 +1,8 @@ # encoding: utf-8 +from __future__ import annotations import itertools +from typing import Any, Optional import click import json @@ -8,18 +10,20 @@ import ckan.logic as logic import ckan.model as model import ckan.plugins as p +from ckan.common import config from ckan.cli import error_shout from ckan.lib.datapreview import ( add_views_to_dataset_resources, get_view_plugins, get_default_view_plugins, ) +from ckan.types import Context _page_size = 100 -@click.group() +@click.group(short_help=u"Manage resource views.") def views(): """Manage resource views. """ @@ -33,18 +37,19 @@ def views(): @click.option(u"-s", u"--search") @click.option(u"-y", u"--yes", is_flag=True) @click.pass_context -def create(ctx, types, dataset, no_default_filters, search, yes): +def create(ctx: click.Context, types: list[str], dataset: list[str], + no_default_filters: bool, search: str, yes: bool): """Create views on relevant resources. You can optionally provide - specific view types (eg `recline_view`, `image_view`). If no types + specific view types (eg `datatables_view`, `image_view`). If no types are provided, the default ones will be used. These are generally the ones defined in the `ckan.views.default_views` config option. - Note that on either case, plugins must be loaded (ie added to + Note that in either case, plugins must be loaded (ie added to `ckan.plugins`), otherwise the command will stop. """ datastore_enabled = ( - u"datastore" in p.toolkit.config[u"ckan.plugins"].split() + u"datastore" in config[u"ckan.plugins"].split() ) flask_app = ctx.meta['flask_app'] @@ -53,7 +58,7 @@ def create(ctx, types, dataset, no_default_filters, search, yes): if loaded_view_plugins is None: return site_user = logic.get_action(u"get_site_user")({u"ignore_auth": True}, {}) - context = {u"user": site_user[u"name"]} + context: Context = {u"user": site_user[u"name"]} page = 1 while True: @@ -116,7 +121,7 @@ def create(ctx, types, dataset, no_default_filters, search, yes): @views.command() @click.argument(u"types", nargs=-1) @click.option(u"-y", u"--yes", is_flag=True) -def clear(types, yes): +def clear(types: list[str], yes: bool): """Permanently delete all views or the ones with the provided types. """ @@ -133,7 +138,7 @@ def clear(types, yes): site_user = logic.get_action(u"get_site_user")({u"ignore_auth": True}, {}) - context = {u"user": site_user[u"name"]} + context: Context = {u"user": site_user[u"name"]} logic.get_action(u"resource_view_clear")(context, {u"view_types": types}) click.secho(u"Done", fg=u"green") @@ -142,7 +147,7 @@ def clear(types, yes): @views.command() @click.option(u"-y", u"--yes", is_flag=True) @click.pass_context -def clean(ctx, yes): +def clean(ctx: click.Context, yes: bool): """Permanently delete views for all types no longer present in the `ckan.plugins` configuration option. @@ -162,16 +167,18 @@ def clean(ctx, yes): for row in results: click.secho(u"%s of type %s" % (row[1], row[0])) - yes or click.confirm( - u"Do you want to delete these resource views?", abort=True - ) + if not yes: + click.confirm( + u"Do you want to delete these resource views?", abort=True + ) model.ResourceView.delete_not_in_view_types(names) model.Session.commit() click.secho(u"Deleted resource views.", fg=u"green") -def _get_view_plugins(view_plugin_types, get_datastore_views=False): +def _get_view_plugins(view_plugin_types: list[str], + get_datastore_views: bool = False): """Returns the view plugins that were succesfully loaded Views are provided as a list of ``view_plugin_types``. If no types @@ -217,13 +224,17 @@ def _get_view_plugins(view_plugin_types, get_datastore_views=False): def _search_datasets( - page=1, view_types=[], dataset=[], search=u"", no_default_filters=False + page: int = 1, view_types: Optional[list[str]] = None, + dataset: Optional[list[str]] = None, search: str = u"", + no_default_filters: bool = False ): """ Perform a query with `package_search` and return the result Results can be paginated using the `page` parameter """ + if not view_types: + view_types = [] n = _page_size @@ -258,12 +269,13 @@ def _search_datasets( if not search_data_dict.get(u"q"): search_data_dict[u"q"] = u"*:*" - query = p.toolkit.get_action(u"package_search")({}, search_data_dict) + query = logic.get_action(u"package_search")({}, search_data_dict) return query -def _add_default_filters(search_data_dict, view_types): +def _add_default_filters(search_data_dict: dict[str, Any], + view_types: list[str]): """ Adds extra filters to the `package_search` dict for common view types @@ -282,20 +294,20 @@ def _add_default_filters(search_data_dict, view_types): modified with extra filters. """ - from ckanext.imageview.plugin import DEFAULT_IMAGE_FORMATS from ckanext.textview.plugin import get_formats as get_text_formats - from ckanext.datapusher.plugin import DEFAULT_FORMATS as datapusher_formats + datapusher_formats = config.get("ckan.datapusher.formats") filter_formats = [] for view_type in view_types: if view_type == u"image_view": - - for _format in DEFAULT_IMAGE_FORMATS: + formats = config.get( + "ckan.preview.image_formats").split() + for _format in formats: filter_formats.extend([_format, _format.upper()]) elif view_type == u"text_view": - formats = get_text_formats(p.toolkit.config) + formats = get_text_formats(config) for _format in itertools.chain.from_iterable(formats.values()): filter_formats.extend([_format, _format.upper()]) @@ -328,7 +340,8 @@ def _add_default_filters(search_data_dict, view_types): return search_data_dict -def _update_search_params(search_data_dict, search): +def _update_search_params( + search_data_dict: dict[str, Any], search: str): """ Update the `package_search` data dict with the user provided parameters diff --git a/ckan/common.py b/ckan/common.py index 28963f3890c..3d8fde56ac8 100644 --- a/ckan/common.py +++ b/ckan/common.py @@ -7,76 +7,75 @@ # # NOTE: This file is specificaly created for # from ckan.common import x, y, z to be allowed +from __future__ import annotations -from collections import MutableMapping +import logging +from collections.abc import MutableMapping, Iterable + +from typing import ( + Any, Optional, TYPE_CHECKING, + TypeVar, cast, overload, Union) +from typing_extensions import Literal import flask -import six from werkzeug.local import Local, LocalProxy +from flask_login import current_user as _cu +from flask_login import login_user as _login_user, logout_user as _logout_user from flask_babel import (gettext as flask_ugettext, ngettext as flask_ungettext) -import simplejson as json +import simplejson as json # type: ignore # noqa: re-export +import ckan.lib.maintain as maintain +from ckan.config.declaration import Declaration +from ckan.types import Model + + +if TYPE_CHECKING: + # starting from python 3.7 the following line can be used without any + # conditions after `annotation` import from `__future__` + MutableMapping = MutableMapping[str, Any] + +SENTINEL = {} + +log = logging.getLogger(__name__) -if six.PY2: - import pylons - from pylons.i18n import (ugettext as pylons_ugettext, - ungettext as pylons_ungettext) - from pylons import response -current_app = flask.current_app +current_user = cast(Union["Model.User", "Model.AnonymousUser"], _cu) +login_user = _login_user +logout_user = _logout_user +@maintain.deprecated('All web requests are served by Flask', since="2.10.0") def is_flask_request(): u''' - A centralized way to determine whether we are in the context of a - request being served by Flask or Pylons + This function is deprecated. All CKAN requests are now served by Flask ''' - if six.PY3: - return True - try: - pylons.request.environ - pylons_request_available = True - except TypeError: - pylons_request_available = False - - return (flask.request and - (flask.request.environ.get(u'ckan.app') == u'flask_app' or - not pylons_request_available)) + return True -def streaming_response( - data, mimetype=u'application/octet-stream', with_context=False): +def streaming_response(data: Iterable[Any], + mimetype: str = u'application/octet-stream', + with_context: bool = False) -> flask.Response: iter_data = iter(data) - if is_flask_request(): - # Removal of context variables for pylon's app is prevented - # inside `pylons_app.py`. It would be better to decide on the fly - # whether we need to preserve context, but it won't affect performance - # in any visible way and we are going to get rid of pylons anyway. - # Flask allows to do this in easy way. - if with_context: - iter_data = flask.stream_with_context(iter_data) - resp = flask.Response(iter_data, mimetype=mimetype) - else: - response.app_iter = iter_data - resp = response.headers['Content-type'] = mimetype + + if with_context: + iter_data = flask.stream_with_context(iter_data) + resp = flask.Response(iter_data, mimetype=mimetype) + return resp -def ugettext(*args, **kwargs): - return flask_ugettext(*args, **kwargs) +def ugettext(*args: Any, **kwargs: Any) -> str: + return cast(str, flask_ugettext(*args, **kwargs)) _ = ugettext -def ungettext(*args, **kwargs): - if is_flask_request(): - return flask_ungettext(*args, **kwargs) - else: - return pylons_ungettext(*args, **kwargs) +def ungettext(*args: Any, **kwargs: Any) -> str: + return cast(str, flask_ungettext(*args, **kwargs)) class CKANConfig(MutableMapping): @@ -89,12 +88,13 @@ class CKANConfig(MutableMapping): `load_environment` method with the values of the ini file or env vars. ''' + store: dict[str, Any] - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any): self.store = dict() self.update(dict(*args, **kwargs)) - def __getitem__(self, key): + def __getitem__(self, key: str): return self.store[key] def __iter__(self): @@ -106,57 +106,47 @@ def __len__(self): def __repr__(self): return self.store.__repr__() - def copy(self): + def copy(self) -> dict[str, Any]: return self.store.copy() - def clear(self): + def clear(self) -> None: self.store.clear() - try: flask.current_app.config.clear() except RuntimeError: pass - if six.PY2: - try: - pylons.config.clear() - # Pylons set this default itself - pylons.config[u'lang'] = None - except TypeError: - pass - - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: Any): self.store[key] = value try: flask.current_app.config[key] = value except RuntimeError: pass - if six.PY2: - try: - pylons.config[key] = value - except TypeError: - pass - - def __delitem__(self, key): + def __delitem__(self, key: str): del self.store[key] try: del flask.current_app.config[key] except RuntimeError: pass - if six.PY2: - try: - del pylons.config[key] - except TypeError: - pass + def is_declared(self, key: str) -> bool: + return key in config_declaration + + def get(self, key: str, default: Any = SENTINEL) -> Any: + """Return the value for key if key is in the config, else default. + """ + if default is SENTINEL: + default = None + is_strict = super().get("config.mode") == "strict" + if is_strict and key not in config_declaration: + log.warning("Option %s is not declared", key) + + return super().get(key, default) def _get_request(): - if is_flask_request(): - return flask.request - else: - return pylons.request + return flask.request class CKANRequest(LocalProxy): @@ -164,59 +154,40 @@ class CKANRequest(LocalProxy): This is just a wrapper around LocalProxy so we can handle some special cases for backwards compatibility. - - LocalProxy will forward to Flask or Pylons own request objects depending - on the output of `_get_request` (which essentially calls - `is_flask_request`) and at the same time provide all objects methods to be - able to interact with them transparently. ''' @property + @maintain.deprecated('Use `request.args` instead of `request.params`', + since="2.10.0") def params(self): - u''' Special case as Pylons' request.params is used all over the place. - All new code meant to be run just in Flask (eg views) should always - use request.args + '''This property is deprecated. + + Special case as Pylons' request.params is used all over the place. All + new code meant to be run just in Flask (eg views) should always use + request.args + ''' - try: - return super(CKANRequest, self).params - except AttributeError: - return self.args + return cast(flask.Request, self).args def _get_c(): - if is_flask_request(): - return flask.g - else: - return pylons.c + return flask.g def _get_session(): - if is_flask_request(): - return flask.session - else: - return pylons.session + return flask.session -local = Local() - -# This a proxy to the bounded config object -local(u'config') +def asbool(obj: Any) -> bool: + """Convert a string (e.g. 1, true, True) into a boolean. -# Thread-local safe objects -config = local.config = CKANConfig() + Example:: -# Proxies to already thread-local safe objects -request = CKANRequest(_get_request) -# Provide a `c` alias for `g` for backwards compatibility -g = c = LocalProxy(_get_c) -session = LocalProxy(_get_session) + assert asbool("yes") is True -truthy = frozenset([u'true', u'yes', u'on', u'y', u't', u'1']) -falsy = frozenset([u'false', u'no', u'off', u'n', u'f', u'0']) + """ - -def asbool(obj): - if isinstance(obj, six.string_types): + if isinstance(obj, str): obj = obj.strip().lower() if obj in truthy: return True @@ -227,22 +198,86 @@ def asbool(obj): return bool(obj) -def asint(obj): +def asint(obj: Any) -> int: + """Convert a string into an int. + + Example:: + + assert asint("111") == 111 + + """ try: return int(obj) except (TypeError, ValueError): raise ValueError(u"Bad integer value: {}".format(obj)) -def aslist(obj, sep=None, strip=True): - if isinstance(obj, six.string_types): +T = TypeVar('T') +SequenceT = TypeVar('SequenceT', "list[Any]", "tuple[Any]") + + +@overload +def aslist(obj: str, + sep: Optional[str] = None, + strip: bool = True) -> list[str]: + ... + + +@overload +def aslist(obj: SequenceT, + sep: Optional[str] = None, + strip: bool = True) -> SequenceT: + ... + + +@overload +def aslist(obj: Literal[None], + sep: Optional[str] = None, + strip: bool = True) -> list[str]: + ... + + +def aslist(obj: Any, sep: Optional[str] = None, strip: bool = True) -> Any: + """Convert a space-separated string into a list. + + Example:: + + assert aslist("a b c") == ["a", "b", "c"] + + """ + + if isinstance(obj, str): lst = obj.split(sep) if strip: lst = [v.strip() for v in lst] return lst elif isinstance(obj, (list, tuple)): - return obj + return cast(Any, obj) + elif isinstance(obj, Iterable): + return list(obj) elif obj is None: return [] else: return [obj] + + +local = Local() + +# This a proxy to the bounded config object +local(u'config') + +# Thread-local safe objects +config = local.config = CKANConfig() + +local("config_declaration") +config_declaration = local.config_declaration = Declaration() + +# Proxies to already thread-local safe objects +request = cast(flask.Request, CKANRequest(_get_request)) +# Provide a `c` alias for `g` for backwards compatibility +g: Any = LocalProxy(_get_c) +c = g +session: Any = LocalProxy(_get_session) + +truthy = frozenset([u'true', u'yes', u'on', u'y', u't', u'1']) +falsy = frozenset([u'false', u'no', u'off', u'n', u'f', u'0']) diff --git a/ckan/config/__init__.py b/ckan/config/__init__.py index e69de29bb2d..40a96afc6ff 100644 --- a/ckan/config/__init__.py +++ b/ckan/config/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/ckan/config/config_declaration.yaml b/ckan/config/config_declaration.yaml new file mode 100644 index 00000000000..7fae5489e67 --- /dev/null +++ b/ckan/config/config_declaration.yaml @@ -0,0 +1,1711 @@ +version: 1 +groups: + # Internal options, that are used/computed by CKAN in runtime + - annotation: ~ + options: + - key: __file__ + internal: true + - key: here + internal: true + - key: plugin_template_paths + ignored: true + - key: plugin_public_paths + ignored: true + - key: computed_template_paths + ignored: true + - key: clear_logo_upload + ignored: true + - key: logo_upload + ignored: true + - key: ckan.host + ignored: true + - key: testing + ignored: true + type: bool + + # Options that are available inside CircleCI containers: + - annotation: ~ + options: + - key: CKAN_POSTGRES_USER + internal: true + - key: CKAN_DATASTORE_POSTGRES_WRITE_USER + internal: true + - key: CKAN_DATASTORE_POSTGRES_READ_USER + internal: true + - key: CKAN_POSTGRES_DB + internal: true + - key: CKAN_DATASTORE_POSTGRES_DB + internal: true + - key: CKAN_DATASTORE_POSTGRES_WRITE_PWD + internal: true + - key: CKAN_POSTGRES_PWD + internal: true + - key: CKAN_DATASTORE_POSTGRES_READ_PWD + internal: true + + # Flask configuration options + - annotation: ~ + options: + - key: APPLICATION_ROOT + internal: true + - key: BABEL_DEFAULT_LOCALE + internal: true + - key: BABEL_DEFAULT_TIMEZONE + ignored: true + - key: BABEL_DOMAIN + ignored: true + - key: BABEL_MULTIPLE_DOMAINS + ignored: true + - key: BABEL_TRANSLATION_DIRECTORIES + ignored: true + - key: CKAN_INI + internal: true + - key: DEBUG + internal: true + - key: ENV + internal: true + - key: EXPLAIN_TEMPLATE_LOADING + internal: true + - key: JSONIFY_MIMETYPE + internal: true + - key: JSONIFY_PRETTYPRINT_REGULAR + internal: true + - key: JSON_AS_ASCII + internal: true + - key: JSON_SORT_KEYS + internal: true + - key: MAX_CONTENT_LENGTH + internal: true + - key: MAX_COOKIE_SIZE + internal: true + - key: PERMANENT_SESSION_LIFETIME + internal: true + - key: PREFERRED_URL_SCHEME + internal: true + - key: PRESERVE_CONTEXT_ON_EXCEPTION + internal: true + - key: PROPAGATE_EXCEPTIONS + internal: true + - key: SECRET_KEY + internal: true + - key: SEND_FILE_MAX_AGE_DEFAULT + internal: true + - key: SERVER_NAME + internal: true + - key: SESSION_COOKIE_DOMAIN + internal: true + - key: SESSION_COOKIE_HTTPONLY + internal: true + - key: SESSION_COOKIE_NAME + internal: true + - key: SESSION_COOKIE_PATH + internal: true + - key: SESSION_COOKIE_SAMESITE + internal: true + - key: SESSION_COOKIE_SECURE + internal: true + - key: SESSION_REFRESH_EACH_REQUEST + internal: true + - key: TEMPLATES_AUTO_RELOAD + internal: true + - key: TESTING + internal: true + - key: TRAP_BAD_REQUEST_ERRORS + internal: true + - key: TRAP_HTTP_EXCEPTIONS + internal: true + - key: USE_X_SENDFILE + internal: true + - key: DEBUG_TB_HOSTS + internal: true + - key: DEBUG_TB_ENABLED + internal: true + - key: DEBUG_TB_INTERCEPT_REDIRECTS + internal: true + - key: DEBUG_TB_PANELS + internal: true + + - annotation: Default settings + section: DEFAULT + options: + - key: debug + type: bool + example: 'true' + description: | + This enables the `Flask-DebugToolbar + `_ in the web interface, makes + Webassets serve unminified JS and CSS files, and enables CKAN templates' + debugging features. + + You will need to ensure the ``Flask-DebugToolbar`` python package is installed, + by activating your ckan virtual environment and then running:: + + pip install -r /usr/lib/ckan/default/src/ckan/dev-requirements.txt + + If you are running CKAN on Apache, you must change the WSGI + configuration to run a single process of CKAN. Otherwise + the execution will fail with: ``AssertionError: The EvalException + middleware is not usable in a multi-process environment``. Eg. change:: + + WSGIDaemonProcess ckan_default display-name=ckan_default processes=2 threads=15 + to + WSGIDaemonProcess ckan_default display-name=ckan_default threads=15 + + .. warning:: This option should be set to ``False`` for a public site. + With debug mode enabled, a visitor to your site could execute malicious + commands. + + - annotation: General settings + options: + - key: use + placeholder: egg:ckan + validators: not_empty + required: true + + - key: ckan.legacy_route_mappings + default: {} + example: '{"home": "home.index", "about": "home.about", "search": "dataset.search"}' + description: | + This can be used when using an extension that is still using old + (Pylons-based) route names to maintain compatibility. + + .. warning:: This configuration will be removed when the migration to + Flask is completed. Please update the extension code to use the new + Flask-based route names. + + - key: config.mode + default: default + example: strict + description: | + Controls the behavior of application when invalid values detected in + the ``config`` object. + + In the ``default`` mode any invalid value is left unprocessed (i.e., + it remains a ``str``). In addition, every invalid option is reported using + a log record with a ``WARNING`` level. + + In the ``strict`` mode, CKAN will not start unless **all** config + options are valid according to the validators defined in the + configuration declaration. For every invalid config option, an error will be + printed to the output stream. + + - annotation: Development settings + options: + - key: ckan.devserver.host + default: localhost + example: '0.0.0.0' + description: Host name to use when running the development server. + - key: ckan.devserver.port + type: int + default: 5000 + example: 5005 + description: Port to use when running the development server. + - key: ckan.devserver.threaded + type: bool + example: 'true' + description: Controls whether the development server should handle each request in a separate thread. + - key: ckan.devserver.multiprocess + type: int + default: 1 + example: 8 + description: | + If greater than 1 then the development server will handle each request in a new process, up to this + maximum number of concurrent processes. + - key: ckan.devserver.watch_patterns + type: list + example: 'mytheme/**/*.yaml mytheme/**/*.json' + description: | + A list of files the reloader should watch to restart the development server, in addition to the + Python modules (for example configuration files) + + - key: ckan.devserver.ssl_cert + example: path/to/host.cert + description: | + Path to a certificate file that will be used to enable SSL (ie to serve the + local development server on https://localhost:5000). You can generate a + self-signed certificate and key (see :ref:`ckan.devserver.ssl_key`) running + the following commands:: + + openssl genrsa 2048 > host.key + chmod 400 host.key + openssl req -new -x509 -nodes -sha256 -days 3650 -key host.key > host.cert + + After that you can run CKAN locally with SSL using this command:: + + ckan -c /path/to/ckan.ini run --ssl-cert=/path/to/host.cert --ssl-key=/path/to/host.key + + Alternatively, setting this option to ``adhoc`` will automatically generate a new + certificate file (on each server reload, which means that you'll get a browser warning + about the certificate on each reload). + + - key: ckan.devserver.ssl_key + example: path/to/host.key + description: | + Path to a certificate file that will be used to enable SSL (ie to serve the + local development server on https://localhost:5000). See :ref:`ckan.devserver.ssl_cert` + for more details. This option also supports the ``adhoc`` value, with the same caveat. + + - annotation: Session settings + options: + - key: ckan.user.last_active_interval + type: int + default: 600 + description: | + The number of seconds between requests to record the last time a user was active on the site. + - key: cache_dir + placeholder: "/tmp/%(ckan.site_id)s" + - key: beaker.session.key + default: ckan + description: Name of the cookie key used to save the session under. + - key: beaker.session.secret + validators: not_empty + required: true + placeholder_callable: secrets:token_urlsafe + callable_args: + nbytes: 20 + description: | + This is the secret token that the beaker library uses to hash the + cookie sent to the client. `ckan generate config` generates a unique + value for this each time it generates a config file. When used in a + cluster environment, the value must be the same on every machine. + - key: beaker.session.auto + type: bool + default: False + description: | + When set to True, the session will save itself anytime it is accessed during a request, + negating the need to issue the save() method. + + - key: beaker.session.cookie_expires + type: bool + description: | + Determines when the cookie used to track the client-side of the session will expire. + When set to a boolean value, it will either expire at the end of the browsers session, or never expire. + Setting to a datetime forces a hard ending time for the session (generally used for setting a session to a far off date). + Setting to an integer will result in the cookie being set to expire in that many seconds. + I.e. a value of 300 will result in the cookie being set to expire in 300 seconds. + Defaults to never expiring. + + - key: beaker.session.cookie_domain + commented: true + placeholder: .example.com + description: | + What domain the cookie should be set to. When using sub-domains, + this should be set to the main domain the cookie should be valid for. For example, + if a cookie should be valid under www.nowhere.com and files.nowhere.com then it should be set to .nowhere.com. + Defaults to the current domain in its entirety. + + - key: beaker.session.save_accessed_time + type: bool + default: True + description: Whether beaker should save the session's access time (true) or only modification time (false). + + - key: beaker.session.secure + type: bool + description: | + Whether or not the session cookie should be marked as secure. When marked as secure, + browsers are instructed to not send the cookie over anything other than an SSL connection. + + - key: beaker.session.timeout + type: int + default: 600 + description: | + Seconds until the session is considered invalid, after which it will be ignored and invalidated. + This number is based on the time since the session was last accessed, not from when the session was created. + Defaults to never expiring. + Requires that save_accessed_time be true. + + - annotation: Database settings + options: + - key: sqlalchemy.url + placeholder: postgresql://ckan_default:pass@localhost/ckan_default + validators: not_empty + required: true + example: postgres://tester:pass@localhost/ckantest3 + description: | + This defines the database that CKAN is to use. The format is:: + + sqlalchemy.url = postgres://USERNAME:PASSWORD@HOST/DBNAME + + - key: sqlalchemy.pool_pre_ping + type: bool + default: true + + - key: sqlalchemy.