diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index d870d0d16a2e..6224abc328cd 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -29,7 +29,7 @@ services: # user: vscode db: - image: "bitnami/postgresql:11.15.0" + image: "bitnamilegacy/postgresql:11.15.0" restart: unless-stopped volumes: - postgres-data:/bitnami/postgresql @@ -47,7 +47,7 @@ services: redis: restart: unless-stopped - image: "bitnami/redis:6.2.7" + image: "bitnamilegacy/redis:6.2.7" environment: REDIS_PASSWORD: "redis" expose: diff --git a/.github/poetry_version.txt b/.github/poetry_version.txt index 8856dd07eb95..271f9b9b1d5e 100644 --- a/.github/poetry_version.txt +++ b/.github/poetry_version.txt @@ -1,2 +1,2 @@ # The poetry version is stored in a separate file due to the https://github.com/python-poetry/poetry/issues/3316 -poetry-version=1.4.2 +poetry-version=1.8.2 diff --git a/.github/workflows/automatic-pr-update.yml b/.github/workflows/automatic-pr-update.yml index 4842a120451b..6621d5985d04 100644 --- a/.github/workflows/automatic-pr-update.yml +++ b/.github/workflows/automatic-pr-update.yml @@ -7,7 +7,7 @@ jobs: # thats's all. single step is needed - if PR is mergeable according to # branch protection rules it will be merged automatically mergepal: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: github.repository == 'RasaHQ/rasa' steps: diff --git a/.github/workflows/automatic-release-to-main-merger.yml b/.github/workflows/automatic-release-to-main-merger.yml deleted file mode 100644 index 6ed4cca20af8..000000000000 --- a/.github/workflows/automatic-release-to-main-merger.yml +++ /dev/null @@ -1,113 +0,0 @@ -name: Automatic main branch merger -on: - # whenever a pull request is merged into a release branch, - # open a pull request to merge changes down to the main branch - pull_request: - branches: - - '[0-9]+.[0-9]+.x' - # Don't merge 2.8.x into main - - '!2.8.x' - # Don't merge 3.0, 3.1 and 3.2 into main - - '!3.0.x' - - '!3.1.x' - - '!3.2.x' - - types: - # means that the PR is closed, we still have to check if it was merged - - closed - -env: - # keep this in sync with the automatic-pr-approver workflow - LABEL_TYPE: type:release-branch-port - LABEL_STATUS: status:ready-to-merge - -jobs: - update_merge_pr: - runs-on: ubuntu-22.04 - - # only run this workflow if a pull request has been merged - # don't run this workflow on pull request from forks, permissions will be missing anyway - # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflows-in-forked-repositories - if: github.event.pull_request.merged == true && github.event.pull_request.head.repo.full_name == 'RasaHQ/rasa' - - steps: - - name: Checkout git repository ๐Ÿ• - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c - - - name: Fetch git tags ๐ŸŽจ - # see https://github.com/actions/checkout/issues/206#issuecomment-617937725 - run: git fetch --prune --unshallow --tags - - - name: Get branch name โœ๏ธ - id: get-branch-name - run: | - GITHUB_BRANCH=${GITHUB_REF/refs\/heads\//} - echo "release_branch=${GITHUB_BRANCH}" >> $GITHUB_OUTPUT - echo "new_branch=merge-${GITHUB_BRANCH}-main-${GITHUB_SHA:0:7}" >> $GITHUB_OUTPUT - - - name: Get GitHub labels ๐Ÿท - id: get-github-labels - run: | - LATEST_RASA_MINOR=$(git tag --list | grep -P '^\d+\.\d+\.\d+$' | tail -n1 | sed -e 's/.\([0-9]\)*$/.0/g') - echo "Latest minor: ${LATEST_RASA_MINOR}" - # bash doesn't support nested variable access - CURRENT_RASA_MINOR=${GITHUB_REF/refs\/heads\//} - CURRENT_RASA_MINOR=${CURRENT_RASA_MINOR/\.x/\.0} - - if [[ ${LATEST_RASA_MINOR} == ${CURRENT_RASA_MINOR} ]] - then - echo "labels=${LABEL_TYPE},${LABEL_STATUS}" >> $GITHUB_OUTPUT - else - echo "labels=${LABEL_TYPE}" >> $GITHUB_OUTPUT - fi - - - name: Create new branch ๐Ÿฃ - id: create-new-branch - if: always() - uses: peterjgrainger/action-create-branch@64aa569aea81305305c6e92bd236d8c427debff8 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - branch: ${{ steps.get-branch-name.outputs.new_branch }} - - - name: Open pull request โ˜„๏ธ - if: ${{ steps.create-new-branch.conclusion == 'success' }} - uses: repo-sync/pull-request@7e79a9f5dc3ad0ce53138f01df2fad14a04831c5 - with: - # using this token to make sure it triggers other actions - github_token: ${{ secrets.RASABOT_GITHUB_TOKEN }} - source_branch: ${{ steps.get-branch-name.outputs.new_branch }} - destination_branch: main - pr_title: Merge ${{ steps.get-branch-name.outputs.release_branch }} into main - pr_template: .github/PULL_REQUEST_AUTOMATIC_TEMPLATE.md - pr_label: ${{ steps.get-github-labels.outputs.labels }} - pr_reviewer: ${{ github.event.pull_request.user.login }} - - - name: Close outdated release-merge PRs ๐Ÿงน - id: close-outdated-release-merge-prs - run: | - # fetch all open merge-PRs that have been opened from the current release branch - gh pr list -S "is:open label:${LABEL_TYPE} head:merge-${{ steps.get-branch-name.outputs.release_branch }}-main" > prs.txt - less prs.txt - - # delete newly opened PR from the list - awk '!/${{ steps.get-branch-name.outputs.new_branch }}/' prs.txt > temp && mv temp prs.txt - - # extract the PR ids - awk '{print $1}' prs.txt > pr_ids.txt - - # close all outdated PRs - while read id; do - gh pr close $id -d - done > $GITHUB_ENV shell: bash - - name: Install poetry ๐Ÿฆ„ - if: needs.changes.outputs.backend == 'true' - uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8 + - name: Install poetry (Ubuntu) ๐Ÿฆ„ + if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-24.04' + uses: snok/install-poetry@93ada01c735cc8a383ce0ce2ae205a21c415379b + with: + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + - name: Install poetry (Windows) ๐Ÿฆ„ + if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2022' + uses: Gr1N/setup-poetry@48b0f77c8c1b1b19cb962f0f00dff7b4be8f81ec #v9 with: poetry-version: ${{ env.POETRY_VERSION }} - - name: Inject setuptools into poetry's runtime environment + - name: Prevent race condition in poetry build + # More context about race condition during poetry build can be found here: + # https://github.com/python-poetry/poetry/issues/7611#issuecomment-1747836233 if: needs.changes.outputs.backend == 'true' run: | - poetry self add setuptools + poetry config installer.max-workers 1 - name: Load Poetry Cached Libraries โฌ‡ id: cache-poetry if: needs.changes.outputs.backend == 'true' - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: .venv key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}-venv-${{ secrets.POETRY_CACHE_VERSION }}-${{ env.pythonLocation }} @@ -316,7 +325,7 @@ jobs: run: poetry config virtualenvs.in-project true - name: Install Dependencies (Linux) ๐Ÿ“ฆ - if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-22.04' + if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-24.04' # Poetry intermittently fails to install dependency if it is not PEP 517 compliant # This is a workaround for that issue run: | @@ -326,7 +335,7 @@ jobs: make prepare-tests-ubuntu - name: Install Dependencies (Windows) ๐Ÿ“ฆ - if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2019' + if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2022' # Restoring cache doesn't work properly on Windows due to symlinks. # We create symlinks for spacy models, that's why we need to clean them up # before caching the dependencies directory. @@ -344,7 +353,7 @@ jobs: make prepare-tests-windows-gha - name: Add github workflow problem matchers - if: needs.changes.outputs.backend == 'true' && matrix.python-version == 3.7 && matrix.os == 'ubuntu-22.04' + if: needs.changes.outputs.backend == 'true' && matrix.python-version == 3.7 && matrix.os == 'ubuntu-24.04' # only annotate based on test runs on ubuntu: otherwise # all errors will be duplicated for each python / os combination # therefore, we only enable for the one where most tests are run @@ -352,7 +361,7 @@ jobs: run: pip install pytest-github-actions-annotate-failures - name: Disable "LongPathsEnabled" option on Windows - if: matrix.os == 'windows-2019' + if: matrix.os == 'windows-2022' # On Windows laptops, a default preset prevents path names from being longer than # 260 characters. Some of our users can't enable this setting due to company policies. # We implemented a fix for model storage. The Windows container in GitHub @@ -363,11 +372,11 @@ jobs: Set-ItemProperty 'HKLM:\System\CurrentControlSet\Control\FileSystem' -Name 'LongPathsEnabled' -value 0 - name: Install ddtrace on Linux - if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-22.04' + if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-24.04' run: poetry run pip install -U 'ddtrace<2.0.0' - name: Install ddtrace on Windows - if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2019' + if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2022' run: | .\.venv\Scripts\activate py -m pip install -U 'ddtrace<2.0.0' @@ -382,18 +391,21 @@ jobs: DD_ARGS: --ddtrace --ddtrace-patch-all run: | make ${{ matrix.test }} - if [[ "${{ matrix.os }}" != "windows-2019" ]]; then - mv .coverage ${{ github.workspace }}/${{ matrix.test }}-coverage - fi shell: bash # bash shell is a way to make code run for both Linux and Windows + - name: Move coverage report + if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-24.04' + run: | + mv .coverage ${{ github.workspace }}/.coverage.${{ matrix.test }} + - name: Store coverage reports - if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-22.04' - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-24.04' + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 #v4.4.3 with: - name: ${{ matrix.test }}-coverage + name: ${{ matrix.test }}-${{ matrix.python-version }}-${{ matrix.os }}-coverage + include-hidden-files: true path: | - ${{ github.workspace }}/${{ matrix.test }}-coverage + ${{ github.workspace }}/.coverage.${{ matrix.test }} test-flaky: name: Run Flaky Tests @@ -404,12 +416,12 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-22.04, windows-2019] + os: [ubuntu-24.04, windows-2022] python-version: [3.8, 3.9, "3.10"] steps: - name: Run DataDog Agent - if: needs.changes.outputs.backend == 'true' && (matrix.os != 'windows-2019' || contains(github.event.pull_request.labels.*.name, 'tools:datadog-windows')) + if: needs.changes.outputs.backend == 'true' && (matrix.os != 'windows-2022' || contains(github.event.pull_request.labels.*.name, 'tools:datadog-windows')) run: | docker run --name dd_agent -p 8126:8126 -d -e "DD_API_KEY=${{ secrets.DD_API_KEY }}" -e "DD_INSIDE_CI=true" -e "DD_HOSTNAME=none" -e "DD_SITE=datadoghq.eu" -e GITHUB_ACTIONS=true -e CI=true datadog/agent:latest docker ps --all --filter name=dd_agent --filter status=running --no-trunc --format "{{.ID}} {{.Status}}" @@ -431,21 +443,25 @@ jobs: echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV shell: bash - - name: Install poetry ๐Ÿฆ„ - if: needs.changes.outputs.backend == 'true' - uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8 + - name: Install poetry (Ubuntu) ๐Ÿฆ„ + if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-24.04' + uses: snok/install-poetry@93ada01c735cc8a383ce0ce2ae205a21c415379b + with: + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + - name: Install poetry (Windows) ๐Ÿฆ„ + if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2022' + uses: Gr1N/setup-poetry@48b0f77c8c1b1b19cb962f0f00dff7b4be8f81ec #v9 with: poetry-version: ${{ env.POETRY_VERSION }} - - name: Inject setuptools into poetry's runtime environment - if: needs.changes.outputs.backend == 'true' - run: | - poetry self add setuptools - - name: Load Poetry Cached Libraries โฌ‡ id: cache-poetry if: needs.changes.outputs.backend == 'true' - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: .venv key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}-venv-${{ secrets.POETRY_CACHE_VERSION }}-${{ env.pythonLocation }} @@ -471,7 +487,7 @@ jobs: run: poetry config virtualenvs.in-project true - name: Install Dependencies (Linux) ๐Ÿ“ฆ - if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-22.04' + if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-24.04' run: | sudo apt-get -y install libpq-dev make install-full | tee .output @@ -479,7 +495,7 @@ jobs: make prepare-tests-ubuntu - name: Install Dependencies (Windows) ๐Ÿ“ฆ - if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2019' + if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2022' # Restoring cache doesn't work properly on Windows due to symlinks. # We create symlinks for spacy models, that's why we need to clean them up # before caching the dependencies' directory. @@ -495,7 +511,7 @@ jobs: make prepare-tests-windows-gha - name: Add github workflow problem matchers - if: needs.changes.outputs.backend == 'true' && matrix.python-version == 3.7 && matrix.os == 'ubuntu-22.04' + if: needs.changes.outputs.backend == 'true' && matrix.python-version == 3.7 && matrix.os == 'ubuntu-24.04' # only annotate based on test runs on ubuntu: otherwise # all errors will be duplicated for each python / os combination # therefore, we only enable for the one where most tests are run @@ -503,7 +519,7 @@ jobs: run: pip install pytest-github-actions-annotate-failures - name: Disable "LongPathsEnabled" option on Windows - if: matrix.os == 'windows-2019' + if: matrix.os == 'windows-2022' # On Windows laptops, a default preset prevents path names from being longer than # 260 characters. Some of our users can't enable this setting due to company policies. # We implemented a fix for model storage. The Windows container in GitHub @@ -514,11 +530,11 @@ jobs: Set-ItemProperty 'HKLM:\System\CurrentControlSet\Control\FileSystem' -Name 'LongPathsEnabled' -value 0 - name: Install ddtrace on Linux - if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-22.04' + if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-24.04' run: poetry run pip install -U 'ddtrace<2.0.0' - name: Install ddtrace on Windows - if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2019' + if: needs.changes.outputs.backend == 'true' && matrix.os == 'windows-2022' run: | .\.venv\Scripts\activate py -m pip install -U 'ddtrace<2.0.0' @@ -533,26 +549,30 @@ jobs: DD_ARGS: --ddtrace --ddtrace-patch-all run: | make test-flaky - if [[ "${{ matrix.os }}" != "windows-2019" ]]; then - mv .coverage ${{ github.workspace }}/test-flaky-coverage - fi shell: bash # bash shell is a way to make code run for both Linux and Windows + - name: Move coverage report + if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-24.04' + run: | + mv .coverage ${{ github.workspace }}/.coverage.test_flaky_coverage + - name: Store coverage reports - if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-22.04' - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + if: needs.changes.outputs.backend == 'true' && matrix.os == 'ubuntu-24.04' + uses: actions/upload-artifact@v4 with: - name: ${{ matrix.test }}-coverage + name: ${{ matrix.test }}-${{ matrix.python-version }}-${{ matrix.os }}-coverage + include-hidden-files: true path: | - ${{ github.workspace }}/${{ matrix.test }}-coverage + ${{ github.workspace }}/.coverage.test_flaky_coverage upload_coverage_reports: name: Upload coverage reports to codeclimate if: github.ref_type != 'tag' - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 # Always upload results even if tests failed needs: - test + - test-flaky - changes steps: @@ -567,25 +587,23 @@ jobs: - name: Get backend coverage reports if: needs.changes.outputs.backend == 'true' - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a + uses: actions/download-artifact@v4 #v4.1.8 with: path: ${{ github.workspace }}/tests_coverage + pattern: "*-coverage" + merge-multiple: true - - name: Merge all reports + - name: List all coverages if: needs.changes.outputs.backend == 'true' run: | - subs=`ls ${{ github.workspace }}/tests_coverage` - download_dir="${{ github.workspace }}/tests_coverage" - final_dir="${{ github.workspace }}/tests_coverage/final" - - # Downloaded artifacts go into folders, gotta extract them all into one folder for upload - mkdir "${final_dir}/" - for i in $subs; do - mv "${download_dir}/$i"/* "${final_dir}/" - done + ls -l ${{ github.workspace }} + ls -l ${{ github.workspace }}/tests_coverage + - name: Merge all reports + if: needs.changes.outputs.backend == 'true' + run: | pip install coverage - coverage combine "${final_dir}/"* + coverage combine ${{ github.workspace }}/tests_coverage/ coverage xml - name: Upload reports to codeclimate @@ -601,7 +619,7 @@ jobs: integration_test: name: Run Non-Sequential Integration Tests if: github.ref_type != 'tag' - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 60 needs: [changes] env: @@ -611,10 +629,6 @@ jobs: POSTGRES_PORT: 5432 POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres - RABBITMQ_HOST: localhost - RABBITMQ_PORT: 5672 - RABBITMQ_USER: guest - RABBITMQ_PASSWORD: guest services: redis: @@ -646,12 +660,6 @@ jobs: # mapping container ports to the host - 5432:5432 - rabbitmq: - # see https://github.com/docker-library/healthcheck/blob/master/rabbitmq/docker-healthcheck - image: healthcheck/rabbitmq - ports: - - 5672:5672 - mongodb: image: mongodb/mongodb-community-server:6.0.4-ubuntu2204 options: >- @@ -681,14 +689,17 @@ jobs: - name: Install poetry ๐Ÿฆ„ if: needs.changes.outputs.backend == 'true' - uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8 + uses: snok/install-poetry@93ada01c735cc8a383ce0ce2ae205a21c415379b with: - poetry-version: ${{ env.POETRY_VERSION }} + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true - name: Load Poetry Cached Libraries โฌ‡ id: cache-poetry if: needs.changes.outputs.backend == 'true' - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: .venv key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}-venv-${{ secrets.POETRY_CACHE_VERSION }}-${{ env.pythonLocation }} @@ -721,24 +732,120 @@ jobs: if grep 'The lock file is not up to date' .output; then exit 1; fi make prepare-tests-ubuntu + - name: Test Code with Services ๐Ÿฉบ + if: needs.changes.outputs.backend == 'true' + env: + JOBS: 2 + INTEGRATION_TEST_PYTEST_MARKERS: '"(not sequential) and (not broker)"' + PYTHONIOENCODING: "utf-8" + run: | + make test-integration + + broker_integration_test: + name: Run Broker Integration Tests + if: github.ref_type != 'tag' + runs-on: ubuntu-24.04 + timeout-minutes: 60 + needs: [changes] + env: + RABBITMQ_HOST: localhost + RABBITMQ_PORT: 5672 + RABBITMQ_USER: guest + RABBITMQ_PASSWORD: guest + + services: + rabbitmq: + # see https://github.com/docker-library/healthcheck/blob/master/rabbitmq/docker-healthcheck + image: healthcheck/rabbitmq + ports: + - 5672:5672 + + steps: + - name: Checkout git repository ๐Ÿ• + if: needs.changes.outputs.backend == 'true' + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c + + - name: Set up Python ${{ env.DEFAULT_PYTHON_VERSION }} ๐Ÿ + if: needs.changes.outputs.backend == 'true' + uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b + with: + python-version: ${{ env.DEFAULT_PYTHON_VERSION }} + + - name: Read Poetry Version ๐Ÿ”ข + if: needs.changes.outputs.backend == 'true' + run: | + echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV + shell: bash + + - name: Install poetry ๐Ÿฆ„ + if: needs.changes.outputs.backend == 'true' + uses: snok/install-poetry@93ada01c735cc8a383ce0ce2ae205a21c415379b + with: + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + - name: Load Poetry Cached Libraries โฌ‡ + id: cache-poetry + if: needs.changes.outputs.backend == 'true' + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: .venv + key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-${{ env.DEFAULT_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }}-venv-${{ secrets.POETRY_CACHE_VERSION }}-${{ env.pythonLocation }} + + - name: Clear Poetry cache + if: steps.cache-poetry.outputs.cache-hit == 'true' && needs.changes.outputs.backend == 'true' && contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-unit-tests') + run: rm -r .venv + + # Poetry >= 1.1.0b uses virtualenv to create a virtual environment. + # The virtualenv simply doesn't work on Windows with our setup, + # that's why we use venv to create virtual environment + - name: Create virtual environment + if: (steps.cache-poetry.outputs.cache-hit != 'true' || contains(github.event.pull_request.labels.*.name, 'tools:clear-poetry-cache-unit-tests')) && needs.changes.outputs.backend == 'true' + run: python -m venv create .venv + + - name: Set up virtual environment + if: needs.changes.outputs.backend == 'true' + # Poetry on Windows cannot pick up the virtual environments directory properly, + # and it creates a new one every time the pipeline runs. + # This step solves this problem โ€” it tells poetry to always use `.venv` directory inside + # the project itself, which also makes it easier for us to determine the correct directory + # that needs to be cached. + run: poetry config virtualenvs.in-project true + + - name: Install Dependencies (Linux) ๐Ÿ“ฆ + if: needs.changes.outputs.backend == 'true' + run: | + sudo apt-get -y install libpq-dev + make install-full | tee .output + if grep 'The lock file is not up to date' .output; then exit 1; fi + make prepare-tests-ubuntu + make prepare-spacy + - name: Run kafka and zookeeper containers for integration testing if: needs.changes.outputs.backend == 'true' run: | - docker-compose -f tests_deployment/docker-compose.kafka.yml up -d + docker compose -f tests_deployment/docker-compose.kafka.yml up -d - name: Test Code with Services ๐Ÿฉบ if: needs.changes.outputs.backend == 'true' env: JOBS: 2 - INTEGRATION_TEST_PYTEST_MARKERS: '"not sequential"' + INTEGRATION_TEST_PYTEST_MARKERS: "broker" PYTHONIOENCODING: "utf-8" run: | make test-integration + - name: Stop kafka and zookeeper containers for integration testing + if: needs.changes.outputs.backend == 'true' + run: | + docker compose -f tests_deployment/docker-compose.kafka.yml down + sequential_integration_test: name: Run Sequential Integration Tests if: github.ref_type != 'tag' - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 60 needs: [changes] env: @@ -783,9 +890,12 @@ jobs: - name: Install poetry ๐Ÿฆ„ if: needs.changes.outputs.backend == 'true' - uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8 + uses: snok/install-poetry@93ada01c735cc8a383ce0ce2ae205a21c415379b with: - poetry-version: ${{ env.POETRY_VERSION }} + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true - name: Load Poetry Cached Libraries โฌ‡ id: cache-poetry @@ -834,14 +944,9 @@ jobs: run: | make test-integration - - name: Stop kafka and zookeeper containers for integration testing - if: needs.changes.outputs.backend == 'true' - run: | - docker-compose -f tests_deployment/docker-compose.kafka.yml down - build_docker_base_images_and_set_env: name: Build Docker base images and setup environment - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: base_image_hash: ${{ steps.check_image.outputs.base_image_hash }} base_mitie_image_hash: ${{ steps.check_image.outputs.base_mitie_image_hash }} @@ -1024,7 +1129,7 @@ jobs: docker: name: Build Docker - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [changes, build_docker_base_images_and_set_env] env: IMAGE_TAG: ${{ needs.build_docker_base_images_and_set_env.outputs.image_tag }} @@ -1126,7 +1231,7 @@ jobs: deploy: name: Deploy to PyPI - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 # deploy will only be run when there is a tag available if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') && github.repository == 'RasaHQ/rasa' @@ -1147,9 +1252,12 @@ jobs: shell: bash - name: Install poetry ๐Ÿฆ„ - uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8 + uses: snok/install-poetry@93ada01c735cc8a383ce0ce2ae205a21c415379b with: - poetry-version: ${{ env.POETRY_VERSION }} + version: ${{ env.POETRY_VERSION }} + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true - name: Copy Segment write key to the package env: @@ -1179,10 +1287,9 @@ jobs: sentry-cli releases set-commits --auto "rasa-$GITHUB_TAG" sentry-cli releases finalize "rasa-$GITHUB_TAG" - - name: Notify Slack & Publish Release Notes ๐Ÿ—ž + - name: Publish Release Notes ๐Ÿ—ž env: GH_RELEASE_NOTES_TOKEN: ${{ secrets.GH_RELEASE_NOTES_TOKEN }} - SLACK_WEBHOOK_TOKEN: ${{ secrets.SLACK_WEBHOOK_TOKEN }} GITHUB_TAG: ${{ github.ref }} GITHUB_REPO_SLUG: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -1190,11 +1297,19 @@ jobs: GITHUB_TAG=${GITHUB_TAG/refs\/tags\//} pip install -U github3.py pep440-version-utils python3 scripts/publish_gh_release_notes.py - ./scripts/ping_slack_about_package_release.sh + + - name: Notify Slack of successful release + # notification will be sent to the #product channel on slack, webhook url is added as repository secret + if: success() + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ASSISTANT_RELEASE_WEBHOOK }} + uses: Ilshidur/action-slack@689ad44a9c9092315abd286d0e3a9a74d31ab78a + with: + args: "๐Ÿ’ฅ New *Rasa Open Source * version `${{ github.ref_name }}` has been released!" send_slack_notification_for_release_on_failure: name: Notify Slack & Publish Release Notes - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 # run this job when the workflow is triggered by a tag push if: always() && github.repository == 'RasaHQ/rasa' && github.ref_type == 'tag' needs: @@ -1205,7 +1320,7 @@ jobs: # send notification if 'deploy' is skipped (previous needed job failed) or failed if: needs.deploy.result != 'success' env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_TOKEN }} + SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ASSISTANT_DEV_TRIBE_WEBHOOK }} uses: Ilshidur/action-slack@689ad44a9c9092315abd286d0e3a9a74d31ab78a with: args: "โ›”๏ธ *Rasa Open Source* version `${{ github.ref_name }}` could not be released ๐Ÿ˜ฑ! Please check out GitHub Actions: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" diff --git a/.github/workflows/dependabot-batch-updater.yml b/.github/workflows/dependabot-batch-updater.yml index 2065260d3bfc..dd08a4cf6c7c 100644 --- a/.github/workflows/dependabot-batch-updater.yml +++ b/.github/workflows/dependabot-batch-updater.yml @@ -6,7 +6,7 @@ on: jobs: update_dependencies: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: Update dependencies steps: - uses: RasaHQ/dependabot-batch-updater@f049cbb0bbd3754bcb5ab154a79f00cd780fc633 # v1.0 diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 3e1a31913f6d..04c536960223 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -30,7 +30,7 @@ env: jobs: changes: name: Check for file changes - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 # don't run this for pull requests of forks if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'RasaHQ/rasa' outputs: @@ -59,7 +59,7 @@ jobs: evaluate_release_tag: name: Evaluate release tag - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 # don't run this for main branches of forks and on documentation branch if: github.repository == 'RasaHQ/rasa' && github.ref != 'refs/heads/documentation' && github.event_name != 'pull_request' outputs: @@ -82,7 +82,7 @@ jobs: echo "build_docs=true" >> $GITHUB_OUTPUT else # Get latest tagged Rasa version - git fetch --depth=1 origin "+refs/tags/*:refs/tags/*" + git describe --tags --match="3.6.[0-9]*" --abbrev=0 HEAD # Fetch branch history TAG_NAME=${GITHUB_REF#refs/tags/} git fetch --prune --unshallow @@ -100,7 +100,7 @@ jobs: prebuild_docs: name: Prebuild Docs - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [evaluate_release_tag] # don't run this for main branches of forks, would fail anyways if: github.repository == 'RasaHQ/rasa' && needs.evaluate_release_tag.outputs.build_docs == 'true' && github.ref != 'refs/heads/documentation' && github.event_name != 'pull_request' @@ -131,7 +131,7 @@ jobs: - name: Load Poetry Cached Libraries โฌ‡ id: cache-poetry - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: .venv key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-3.9-non-full-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }} @@ -149,7 +149,7 @@ jobs: run: poetry config virtualenvs.in-project true - name: Load Yarn Cached Packages โฌ‡ - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: docs/node_modules key: ${{ runner.os }}-yarn-12.x-${{ hashFiles('docs/yarn.lock') }} @@ -189,7 +189,7 @@ jobs: preview_docs: name: Preview Docs - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [changes] # don't run this for pull requests from forks if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'RasaHQ/rasa' @@ -226,7 +226,7 @@ jobs: - name: Load Poetry Cached Libraries โฌ‡ id: cache-poetry if: needs.changes.outputs.docs == 'true' - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: .venv key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-3.9-non-full-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }} @@ -246,7 +246,7 @@ jobs: - name: Load Yarn Cached Packages โฌ‡ if: needs.changes.outputs.docs == 'true' - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: docs/node_modules key: ${{ runner.os }}-yarn-12.x-${{ hashFiles('docs/yarn.lock') }} @@ -285,7 +285,7 @@ jobs: publish_docs: name: Publish Docs - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 # don't run this for main branches of forks; only run on documentation branch if: github.repository == 'RasaHQ/rasa' && github.ref == 'refs/heads/documentation' @@ -299,7 +299,7 @@ jobs: node-version: "12.x" - name: Load Yarn Cached Packages โฌ‡ - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: docs/node_modules key: ${{ runner.os }}-yarn-12.x-${{ hashFiles('docs/yarn.lock') }} diff --git a/.github/workflows/nightly_release.yml b/.github/workflows/nightly_release.yml index c7639d0cbf16..772f06a8a48a 100644 --- a/.github/workflows/nightly_release.yml +++ b/.github/workflows/nightly_release.yml @@ -9,7 +9,7 @@ on: jobs: run_script_and_tag_nightly_release: name: Run release script and tag a new nightly release - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: tag_name: ${{ steps.set_tagname.outputs.tag_name }} @@ -52,7 +52,7 @@ jobs: deploy: name: Deploy to PyPI - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 # deploy will only be run when there is a tag available needs: run_script_and_tag_nightly_release # only run after all other stages succeeded @@ -112,7 +112,7 @@ jobs: docker: name: Build Docker - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: run_script_and_tag_nightly_release env: GCLOUD_VERSION: "297.0.1" diff --git a/.github/workflows/rasa-install-cron-check.yml b/.github/workflows/rasa-install-cron-check.yml index 91fa6eb6ee1d..2cc143c35681 100644 --- a/.github/workflows/rasa-install-cron-check.yml +++ b/.github/workflows/rasa-install-cron-check.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: os: - [ubuntu-22.04, ubuntu-18.04, macos-latest, windows-2019, windows-2022] + [ubuntu-24.04, ubuntu-18.04, macos-latest, windows-2019, windows-2022] python-version: [3.8, 3.9, '3.10'] fail-fast: false diff --git a/.github/workflows/security-scans.yml b/.github/workflows/security-scans.yml index 7348db48311f..612518e1db1a 100644 --- a/.github/workflows/security-scans.yml +++ b/.github/workflows/security-scans.yml @@ -11,7 +11,7 @@ concurrency: jobs: changes: name: Check for file changes - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: backend: ${{ steps.filter.outputs.backend }} docker: ${{ steps.filter.outputs.docker }} @@ -26,7 +26,7 @@ jobs: trivy: name: Detecting hardcoded secrets - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c with: @@ -75,7 +75,7 @@ jobs: bandit: name: Detect python security issues - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [changes] steps: @@ -104,7 +104,7 @@ jobs: - name: Load Poetry Cached Libraries โฌ‡ id: cache-poetry if: needs.changes.outputs.backend == 'true' - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: .venv key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-3.9-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }} @@ -129,24 +129,3 @@ jobs: - name: Run Bandit ๐Ÿ”ช if: needs.changes.outputs.backend == 'true' run: make lint-security - - snyk: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c - - name: Run Snyk Open Source to check for Python vulnerabilities - uses: snyk/actions/python-3.8@master - continue-on-error: true - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - with: - command: monitor - args: --all-projects --org=rasa --skip-unresolved - - name: Run Snyk Open Source to check for JS vulnerabilities - uses: snyk/actions/node@master - continue-on-error: true - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - with: - command: monitor - args: --org=rasa --yarn-workspaces --strict-out-of-sync=false --prune-repeated-subdependencies diff --git a/.github/workflows/semgrep-check.yml b/.github/workflows/semgrep-check.yml index 846f92494b26..0e6665738406 100644 --- a/.github/workflows/semgrep-check.yml +++ b/.github/workflows/semgrep-check.yml @@ -14,7 +14,7 @@ jobs: # User-definable name of this GitHub Actions job: name: Semgrep Workflow Security Scan # If you are self-hosting, change the following `runs-on` value: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 container: # A Docker image with Semgrep installed. Do not change this. diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index 4f90d2724ca9..3359884634e9 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -7,7 +7,7 @@ on: jobs: spellcheck: name: Typo CI (GitHub Action) - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 4 if: "!contains(github.event.head_commit.message, '[ci skip]')" steps: diff --git a/CHANGELOG.mdx b/CHANGELOG.mdx index 9a6177e3632c..d1290c72e19e 100644 --- a/CHANGELOG.mdx +++ b/CHANGELOG.mdx @@ -16,24 +16,102 @@ https://github.com/RasaHQ/rasa/tree/main/changelog/ . --> -## [3.6.13] - 2023-10-23 +## [3.6.21] - 2025-01-13 + +Rasa 3.6.21 (2025-01-13) +### Bugfixes +- [#1424](https://github.com/rasahq/rasa/issues/1424): Replace `pickle` and `joblib` with safer alternatives, e.g. `json`, `safetensors`, and `skops`, for + serializing components. + + **Note**: This is a model breaking change. Please retrain your model. + + If you have a custom component that inherits from one of the components listed below and modified the `persist` or + `load` method, make sure to update your code. Please contact us in case you encounter any problems. + + Affected components: + + - `CountVectorFeaturizer` + - `LexicalSyntacticFeaturizer` + - `LogisticRegressionClassifier` + - `SklearnIntentClassifier` + - `DIETClassifier` + - `CRFEntityExtractor` + - `TrackerFeaturizer` + - `TEDPolicy` + - `UnexpectedIntentTEDPolicy` + + +## [3.6.20] - 2024-04-18 + +Rasa 3.6.20 (2024-04-18) +### Miscellaneous internal changes +- [#13030](https://github.com/rasahq/rasa/issues/13030), [#13031](https://github.com/rasahq/rasa/issues/13031) + + +## [3.6.19] - 2024-03-04 + +Rasa 3.6.19 (2024-03-04) +### Bugfixes +- [#13019](https://github.com/rasahq/rasa/issues/13019): Changed the ordering of returned events to order by ID (previously timestamp) in SQL Tracker Store + + +## [3.6.18] - 2024-02-23 + +Rasa 3.6.18 (2024-02-23) +### Bugfixes +- [#13017](https://github.com/rasahq/rasa/issues/13017): Flush messages when Kafka producer is closed. This is to ensure that all messages in the producer's internal queue are sent to the broker. + + +## [3.6.17] - 2024-02-13 + +Rasa 3.6.17 (2024-02-13) +### Miscellaneous internal changes +- [#13007](https://github.com/rasahq/rasa/issues/13007) + + +## [3.6.16] - 2024-01-19 -Rasa 3.6.13 (2023-10-23) +Rasa 3.6.16 (2024-01-19) +### Bugfixes +- [#12983](https://github.com/rasahq/rasa/issues/12983): Upgrade Cryptography to fix improper certificate validation. +- [#12998](https://github.com/rasahq/rasa/issues/12998): Fixes a bug that caused the `full_retrieval_intent_name` key to be missing in the published event. Rasa Analytics makes use of this key to get the Retrieval Intent Name + +### Miscellaneous internal changes +- [#712](https://github.com/rasahq/rasa/issues/712) + + +## [3.6.15] - 2023-11-30 + +Rasa 3.6.15 (2023-11-30) +### Bugfixes +- [#12965](https://github.com/rasahq/rasa/issues/12965): Fixed connection timeout to action server by setting KEEP_ALIVE_TIMEOUT to 120, and reverting changes introduced in #12886. + + +## [3.6.14] - 2023-11-17 + +Rasa 3.6.14 (2023-11-17) +### Bugfixes +- [#12948](https://github.com/rasahq/rasa/issues/12948): Fixed UnexpecTEDIntentlessPolicy training errors that resulted from a change to batching behavior. Changed the batching behavior back to the original for all components. Made the changed batching behavior accessible in DietClassifier using `drop_small_last_batch: True`. + + +## [3.6.13] - 2023-10-23 + +Rasa 3.6.13 (2023-10-23) ### Bugfixes - [#12927](https://github.com/rasahq/rasa/issues/12927): Fix wrong conflicts that occur when rasa validate stories is run with slots that have active_loop set to null in mapping conditions. ## [3.6.12] - 2023-10-10 - -Rasa 3.6.12 (2023-10-10) + +Rasa 3.6.12 (2023-10-10) ### Bugfixes - [#12904](https://github.com/rasahq/rasa/issues/12904): Refresh headers used in requests (e.g. action server requests) made by `EndpointConfig` using its `headers` attribute. - [#12906](https://github.com/rasahq/rasa/issues/12906): Upgrade `pillow` to `10.0.1` to address security vulnerability CVE-2023-4863 found in `10.0.0` version. ## [3.6.11] - 2023-10-05 - -Rasa 3.6.11 (2023-10-05) + +Rasa 3.6.11 (2023-10-05) ### Bugfixes - [#12722](https://github.com/rasahq/rasa/issues/12722): Intent names will not be falsely abbreviated in interactive training (fixes OSS-413). @@ -45,8 +123,8 @@ Rasa 3.6.11 (2023-10-05) ## [3.6.10] - 2023-09-26 - -Rasa 3.6.10 (2023-09-26) + +Rasa 3.6.10 (2023-09-26) ### Improvements - [#12827](https://github.com/rasahq/rasa/issues/12827): Improved handling of last batch during DIET and TED training. The last batch is discarded if it contains less than half a batch size of data. - [#12852](https://github.com/rasahq/rasa/issues/12852): Added `username` to the connection parameters for `RedisLockStore` and `RedisTrackerStore` @@ -57,8 +135,8 @@ Rasa 3.6.10 (2023-09-26) ## [3.6.9] - 2023-09-15 - -Rasa 3.6.9 (2023-09-15) + +Rasa 3.6.9 (2023-09-15) ### Improvements - [#12778](https://github.com/rasahq/rasa/issues/12778): Added additional method `fingerprint_addon` to the `GraphComponent` interface to allow inclusion of external data into the fingerprint calculation of a component @@ -67,29 +145,29 @@ Rasa 3.6.9 (2023-09-15) ## [3.6.8] - 2023-08-30 - -Rasa 3.6.8 (2023-08-30) + +Rasa 3.6.8 (2023-08-30) No significant changes. ## [3.6.7] - 2023-08-29 - -Rasa 3.6.7 (2023-08-29) + +Rasa 3.6.7 (2023-08-29) ### Bugfixes - [#12768](https://github.com/rasahq/rasa/issues/12768): Updated certifi, cryptography, and scipy packages to address security vulnerabilities. ## [3.6.6] - 2023-08-23 - -Rasa 3.6.6 (2023-08-23) + +Rasa 3.6.6 (2023-08-23) ### Bugfixes - [#12755](https://github.com/rasahq/rasa/issues/12755): Updated setuptools and wheel to address security vulnerabilities. ## [3.6.5] - 2023-08-17 - -Rasa 3.6.5 (2023-08-17) + +Rasa 3.6.5 (2023-08-17) ### Improvements - [#12696](https://github.com/rasahq/rasa/issues/12696): Use the same session across requests in `RasaNLUHttpInterpreter` @@ -102,8 +180,8 @@ Rasa 3.6.5 (2023-08-17) ## [3.6.4] - 2023-07-21 - -Rasa 3.6.4 (2023-07-21) + +Rasa 3.6.4 (2023-07-21) ### Bugfixes - [#12575](https://github.com/rasahq/rasa/issues/12575): Extract conditional response variation and channel variation filtering logic into a separate component. Enable usage of this component in the NaturalLanguageGenerator subclasses (e.g. CallbackNaturalLanguageGenerator, TemplatedNaturalLanguageGenerator). @@ -114,8 +192,8 @@ Rasa 3.6.4 (2023-07-21) ## [3.6.3] - 2023-07-20 - -Rasa 3.6.3 (2023-07-20) + +Rasa 3.6.3 (2023-07-20) ### Improvements - [#12637](https://github.com/rasahq/rasa/issues/12637): Added a human readable component to structlog using the `event_info` key and made it the default rendered key if present. @@ -130,15 +208,15 @@ Rasa 3.6.3 (2023-07-20) ## [3.6.2] - 2023-07-06 - -Rasa 3.6.2 (2023-07-06) + +Rasa 3.6.2 (2023-07-06) ### Bugfixes - [#12602](https://github.com/rasahq/rasa/issues/12602): Resolves the issue of importing TensorFlow on Docker for ARM64 architecture. ## [3.6.1] - 2023-07-03 - -Rasa 3.6.1 (2023-07-03) + +Rasa 3.6.1 (2023-07-03) ### Improvements - [#12533](https://github.com/rasahq/rasa/issues/12533): Add building multi-platform Docker image (amd64/arm64) - [#12543](https://github.com/rasahq/rasa/issues/12543): Switch struct log to `FilteringBoundLogger` in order to retain log level set in the config. @@ -160,8 +238,8 @@ Rasa 3.6.1 (2023-07-03) ## [3.6.0] - 2023-06-14 - -Rasa 3.6.0 (2023-06-14) + +Rasa 3.6.0 (2023-06-14) ### Deprecations and Removals - [#12355](https://github.com/rasahq/rasa/issues/12355): Removed Python 3.7 support as [it reaches its end of life in June 2023](https://devguide.python.org/versions/) @@ -205,6 +283,55 @@ Rasa 3.6.0 (2023-06-14) ### Miscellaneous internal changes - [#12291](https://github.com/rasahq/rasa/issues/12291), [#12329](https://github.com/rasahq/rasa/issues/12329), [#12332](https://github.com/rasahq/rasa/issues/12332), [#12365](https://github.com/rasahq/rasa/issues/12365), [#12372](https://github.com/rasahq/rasa/issues/12372), [#12386](https://github.com/rasahq/rasa/issues/12386), [#12492](https://github.com/rasahq/rasa/issues/12492) +## [3.5.17] - 2023-12-05 + +Rasa 3.5.17 (2023-12-05) +### Improvements +- [#12851](https://github.com/rasahq/rasa/issues/12851): Added `username` to the connection parameters for `RedisLockStore` and `RedisTrackerStore` +- [#1493](https://github.com/rasahq/rasa/issues/1493): Telemetry data is only send for licensed users. + + +## [3.5.16] - 2023-08-30 + +Rasa 3.5.16 (2023-08-30) + +No significant changes. + + +## [3.5.15] - 2023-07-21 + +Rasa 3.5.15 (2023-07-21) + +No significant changes. + + +## [3.5.14] - 2023-07-12 + +Rasa 3.5.14 (2023-07-12) +### Bugfixes +- [#12639](https://github.com/rasahq/rasa/issues/12639): Fix the issue with the most recent model not being selected if the owner or permissions where modified on the model file. + +### Miscellaneous internal changes +- [#12649](https://github.com/rasahq/rasa/issues/12649) + + +## [3.5.13] - 2023-07-05 + +Rasa 3.5.13 (2023-07-05) +### Bugfixes +- [#12549](https://github.com/rasahq/rasa/issues/12549): Introduce a validation step in `rasa data validate` command to identify non-existent paths and empty domains. + + +## [3.5.12] - 2023-06-23 + +Rasa 3.5.12 (2023-06-23) +### Bugfixes +- [#12534](https://github.com/rasahq/rasa/issues/12534): Rich responses containing buttons with parentheses characters are now correctly parsed. + Previously any characters found between the first identified pair of `()` in response button took precedence. + +### Miscellaneous internal changes +- [#12512](https://github.com/rasahq/rasa/issues/12512) + ## [3.5.11] - 2023-06-08 diff --git a/Makefile b/Makefile index f3ebb6135e25..a58eda4a3939 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ JOBS ?= 1 INTEGRATION_TEST_FOLDER = tests/integration_tests/ -INTEGRATION_TEST_PYTEST_MARKERS ?= "sequential or not sequential" +INTEGRATION_TEST_PYTEST_MARKERS ?= "sequential or broker or ((not sequential) and (not broker))" PLATFORM ?= "linux/amd64" help: @@ -127,7 +127,7 @@ prepare-tests-macos: brew install wget graphviz || true prepare-tests-ubuntu: - sudo apt-get -y install graphviz graphviz-dev python-tk + sudo apt-get update && sudo apt-get -y install graphviz graphviz-dev python3-tk prepare-tests-windows: choco install wget graphviz @@ -136,7 +136,7 @@ prepare-tests-windows: # It will retry the installation 5 times if it fails # See: https://github.com/actions/virtual-environments/blob/main/images/win/scripts/ImageHelpers/ChocoHelpers.ps1 prepare-tests-windows-gha: - powershell -command "Choco-Install wget graphviz" + powershell -command "Install-ChocoPackage wget graphviz" test: clean # OMP_NUM_THREADS can improve overall performance using one thread by process (on tensorflow), avoiding overload diff --git a/README.md b/README.md index a9dae57dce95..c535027d142b 100644 --- a/README.md +++ b/README.md @@ -2,450 +2,133 @@
-[![Join the chat on Rasa Community Forum](https://img.shields.io/badge/forum-join%20discussions-brightgreen.svg)](https://forum.rasa.com/?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![Join the Agent Engineering Community](https://img.shields.io/badge/Community-Join%20the%20Discussion-blueviolet)](https://info.rasa.com/community?utm_source=github&utm_medium=website&utm_campaign=) +[![Try Hello Rasa](https://img.shields.io/badge/Playground-Try%20Hello%20Rasa-ff69b4)](https://hello.rasa.ai/?utm_source=github&utm_medium=website&utm_campaign=) [![PyPI version](https://badge.fury.io/py/rasa.svg)](https://badge.fury.io/py/rasa) [![Supported Python Versions](https://img.shields.io/pypi/pyversions/rasa.svg)](https://pypi.python.org/pypi/rasa) [![Build Status](https://github.com/RasaHQ/rasa/workflows/Continuous%20Integration/badge.svg)](https://github.com/RasaHQ/rasa/actions) -[![Coverage Status](https://api.codeclimate.com/v1/badges/756dc6fea1d5d3e127f7/test_coverage)](https://codeclimate.com/github/RasaHQ/rasa/) [![Documentation Status](https://img.shields.io/badge/docs-stable-brightgreen.svg)](https://rasa.com/docs) -![Documentation Build](https://img.shields.io/netlify/d2e447e4-5a5e-4dc7-be5d-7c04ae7ff706?label=Documentation%20Build) -[![FOSSA Status](https://app.fossa.com/api/projects/custom%2B8141%2Fgit%40github.com%3ARasaHQ%2Frasa.git.svg?type=shield)](https://app.fossa.com/projects/custom%2B8141%2Fgit%40github.com%3ARasaHQ%2Frasa.git?ref=badge_shield) -[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](https://github.com/orgs/RasaHQ/projects/23)
-
- -๐Ÿ’ก **We're migrating issues to Jira** ๐Ÿ’ก +
-Starting January 2023, issues for Rasa Open Source are located in -[this Jira board](https://rasa-open-source.atlassian.net/browse/OSS). You can browse issues without being logged in; -if you want to create issues, you'll need to create a Jira account. +
+

๐Ÿšง Note: Maintenance Mode ๐Ÿšง

+

+ Rasa Open Source is currently in maintenance mode. +
+ The future of building AI agents with Rasa is Hello Rasa and CALM. +

+
-An image of Sara, the Rasa mascot bird, holding a flag that reads Open Source with one wing, and a wrench in the other - -Rasa is an open source machine learning framework to automate text and voice-based conversations. With Rasa, you can build contextual assistants on: -- Facebook Messenger -- Slack -- Google Hangouts -- Webex Teams -- Microsoft Bot Framework -- Rocket.Chat -- Mattermost -- Telegram -- Twilio -- Your own custom conversational channels - -or voice assistants as: -- Alexa Skills -- Google Home Actions +## ๐Ÿš€ The Future of Rasa: Hello Rasa -Rasa helps you build contextual assistants capable of having layered conversations with -lots of back-and-forth. In order for a human to have a meaningful exchange with a contextual -assistant, the assistant needs to be able to use context to build on things that were previously -discussed โ€“ Rasa enables you to build assistants that can do this in a scalable way. +**Building reliable AI agents just got easier.** -There's a lot more background information in this -[blog post](https://medium.com/rasa-blog/a-new-approach-to-conversational-software-2e64a5d05f2a). +[**Hello Rasa**](https://hello.rasa.ai/?utm_source=github&utm_medium=website&utm_campaign=) is our new interactive playground for prototyping AI agents. It combines LLM fluency with the reliability of business logic using our **CALM** (Conversational AI with Language Models) engine. ---- -- ๐Ÿค” [Learn more about Rasa](https://rasa.community/) +### Why switch to Hello Rasa? -- ๐Ÿค“ [Read The Docs](https://rasa.com/docs/rasa/) +* **No setup required:** Open the playground, pick a template (Banking, Telecom, Support), and start building in your browser. +* **No NLU training:** We have moved beyond intents. The LLM handles dialogue understanding while you define the business flows. +* **Built-in copilot:** A specialized AI assistant helps you generate code, debug flows, and expand your agent instantly. +* **Production ready:** Hello Rasa is not just a toy. Export your agent to the Rasa Platform when you are ready to scale. -- ๐Ÿ˜ [Install Rasa](https://rasa.com/docs/rasa/installation/environment-set-up) - -- ๐Ÿš€ [Dive deeper in the learning center](https://learning.rasa.com/) - -- ๐Ÿค— [Contribute](#how-to-contribute) - -- โ“ [Get enterprise-grade support](https://rasa.com/support/) - -- ๐Ÿข [Explore the features of our commercial platform](https://rasa.com/product/rasa-platform/) - -- ๐Ÿ“š [Learn more about research papers that leverage Rasa](https://scholar.google.com/scholar?oi=bibs&hl=en&authuser=1&cites=16243802403383697687,353275993797024115,14567308604105196228,9067977709825839723,9855847065463746011&as_sdt=5) +### Core concepts +* **CALM:** Combines LLM flexibility with strict business logic. The LLM understands the user; the code enforces the rules. +* **Flows:** Describe logical steps (e.g., collect money, transfer funds) rather than rigid dialogue trees. +* **Inspector:** See real-time decision-making. No black boxes. +๐Ÿ‘‰ **[Start building for free at Hello Rasa](https://hello.rasa.ai/?utm_source=github&utm_medium=website&utm_campaign=)** --- -## Where to get help - -There is extensive documentation in the [Rasa Docs](https://rasa.com/docs/rasa). -Make sure to select the correct version so you are looking at -the docs for the version you installed. - -Please use [Rasa Community Forum](https://forum.rasa.com) for quick answers to -questions. - -### README Contents: -- [How to contribute](#how-to-contribute) -- [Development Internals](#development-internals) -- [Releases](#releases) -- [License](#license) - -### How to contribute -We are very happy to receive and merge your contributions into this repository! - -To contribute via pull request, follow these steps: - -1. Create an issue describing the feature you want to work on (or - have a look at the [contributor board](https://github.com/orgs/RasaHQ/projects/23)) -2. Write your code, tests and documentation, and format them with ``black`` -3. Create a pull request describing your changes - -For more detailed instructions on how to contribute code, check out these [code contributor guidelines](CONTRIBUTING.md). - -You can find more information about how to contribute to Rasa (in lots of -different ways!) [on our website.](http://rasa.community). - -Your pull request will be reviewed by a maintainer, who will get -back to you about any necessary changes or questions. You will -also be asked to sign a -[Contributor License Agreement](https://cla-assistant.io/RasaHQ/rasa). - - -## Development Internals - -### Installing Poetry -Rasa uses Poetry for packaging and dependency management. If you want to build it from source, -you have to install Poetry first. Please follow -[the official guide](https://python-poetry.org/docs/#installation) to see all possible options. +## ๐Ÿง  Join the Agent Engineering Community -To update an existing poetry version to the [version](.github/poetry_version.txt), currently used in rasa, run: -```shell - poetry self update -``` - -### Managing environments - -The official [Poetry guide](https://python-poetry.org/docs/managing-environments/) suggests to use -[pyenv](https://github.com/pyenv/pyenv) or any other similar tool to easily switch between Python versions. -This is how it can be done: - -```bash -pyenv install 3.10.10 -pyenv local 3.10.10 # Activate Python 3.10.10 for the current project -``` -*Note*: If you have trouble installing a specific version of python on your system -it might be worth trying other supported versions. - -By default, Poetry will try to use the currently activated Python version to create the virtual environment -for the current project automatically. You can also create and activate a virtual environment manually โ€” in this -case, Poetry should pick it up and use it to install the dependencies. For example: - -```bash -python -m venv .venv -source .venv/bin/activate -``` - -You can make sure that the environment is picked up by executing - -```bash -poetry env info -``` +We are building a home for people shipping real-world AI agents. -### Building from source - -To install dependencies and `rasa` itself in editable mode execute - -```bash -make install -``` - -*Note for macOS users*: under macOS Big Sur we've seen some compiler issues for -dependencies. Using `export SYSTEM_VERSION_COMPAT=1` before the installation helped. - - -#### Installing optional dependencies - -In order to install rasa's optional dependencies, you need to run: - -```bash -make install-full -``` - -*Note for macOS users*: The command `make install-full` could result in a failure while installing `tokenizers` -(issue described in depth [here](https://github.com/huggingface/tokenizers/issues/1050)). - -In order to resolve it, you must follow these steps to install a Rust compiler: -```bash -brew install rustup -rustup-init -``` - -After initialising the Rust compiler, you should restart the console and check its installation: -```bash -rustc --version -``` - -In case the PATH variable had not been automatically setup, run: -```bash -export PATH="$HOME/.cargo/bin:$PATH" -``` +Agent Engineering is evolving faster than any single framework. This is a vendor-neutral space to discuss architectures, memory, orchestration, and safety with builders across the industry. +### What you get: +* **Network:** Meet engineers building production agents +* **Learn:** Discuss practical patterns, not just theory +* **Access:** Direct influence on the Hello Rasa roadmap and early access to features -### Running and changing the documentation +| Channel | Purpose | +| :--- | :--- | +| **#agent-design** | Architectures, reasoning, memory, testing | +| **#showcase** | Show your builds, demos, and repos | +| **#ask-anything** | Debugging and workflow questions | -First of all, install all the required dependencies: +๐Ÿ‘‰ **[Join the Community](https://info.rasa.com/community?utm_source=github&utm_medium=website&utm_campaign=)** -```bash -make install install-docs -``` - -After the installation has finished, you can run and view the documentation -locally using: - -```bash -make livedocs -``` - -It should open a new tab with the local version of the docs in your browser; -if not, visit http://localhost:3000 in your browser. -You can now change the docs locally and the web page will automatically reload -and apply your changes. - -### Running the Tests - -In order to run the tests, make sure that you have the development requirements installed: - -```bash -make prepare-tests-ubuntu # Only on Ubuntu and Debian based systems -make prepare-tests-macos # Only on macOS -``` - -Then, run the tests: - -```bash -make test -``` - -They can also be run at multiple jobs to save some time: - -```bash -JOBS=[n] make test -``` - -Where `[n]` is the number of jobs desired. If omitted, `[n]` will be automatically chosen by pytest. - - -### Running the Integration Tests - -In order to run the integration tests, make sure that you have the development requirements installed: - -```bash -make prepare-tests-ubuntu # Only on Ubuntu and Debian based systems -make prepare-tests-macos # Only on macOS -``` - -Then, you'll need to start services with the following command which uses -[Docker Compose](https://docs.docker.com/compose/install/): - -```bash -make run-integration-containers -``` - -Finally, you can run the integration tests like this: - -```bash -make test-integration -``` - - -### Resolving merge conflicts - -Poetry doesn't include any solution that can help to resolve merge conflicts in -the lock file `poetry.lock` by default. -However, there is a great tool called [poetry-merge-lock](https://poetry-merge-lock.readthedocs.io/en/latest/). -Here is how you can install it: - -```bash -pip install poetry-merge-lock -``` - -Just execute this command to resolve merge conflicts in `poetry.lock` automatically: - -```bash -poetry-merge-lock -``` - -### Build a Docker image locally +--- -In order to build a Docker image on your local machine execute the following command: +
+
-```bash -make build-docker -``` +# Rasa Open Source (Legacy) -The Docker image is available on your local machine as `rasa:localdev`. +> **Note:** The documentation and installation instructions below apply to the classic Rasa Open Source framework. For the latest CALM-based experience, see the [Hello Rasa](#-the-future-of-rasa-hello-rasa) section above. -### Code Style +Rasa is an open source machine learning framework for automating text and voice-based conversations. With Rasa, you can build contextual assistants on: -To ensure a standardized code style we use the formatter [black](https://github.com/ambv/black). -To ensure our type annotations are correct we use the type checker [pytype](https://github.com/google/pytype). -If your code is not formatted properly or doesn't type check, GitHub will fail to build. +- Facebook Messenger +- Slack +- Google Hangouts +- Webex Teams +- Microsoft Bot Framework +- Rocket.Chat +- Mattermost +- Telegram +- Twilio +- Your own custom conversational channels -#### Formatting +Rasa helps you build contextual assistants that can handle layered conversations with lots of back-and-forth. -If you want to automatically format your code on every commit, you can use [pre-commit](https://pre-commit.com/). -Just install it via `pip install pre-commit` and execute `pre-commit install` in the root folder. -This will add a hook to the repository, which reformats files on every commit. +### ๐Ÿ“š Resources +- ๐Ÿค“ [Read the docs](https://rasa.com/docs/rasa/) +- ๐Ÿ˜ [Install Rasa](https://rasa.com/docs/rasa/installation/environment-set-up) +- ๐Ÿš€ [Learn all about Conversational AI](https://learning.rasa.com/) +- ๐Ÿข [Explore the enterprise platform](https://rasa.com/product/rasa-platform/) -If you want to set it up manually, install black via `poetry install`. -To reformat files execute -``` -make formatter -``` +## Development Internals & Contributing -#### Type Checking +We are happy to receive contributions. Please review our [Contribution Guidelines](CONTRIBUTING.md) before getting started. -If you want to check types on the codebase, install `mypy` using `poetry install`. -To check the types execute -``` -make types -``` +### Installation for Development +Rasa uses **Poetry** for packaging and dependency management. -### Deploying documentation updates - -We use `Docusaurus v2` to build docs for tagged versions and for the `main` branch. -To run Docusaurus, install `Node.js 12.x`. -The static site that gets built is pushed to the `documentation` branch of this repo. - -We host the site on netlify. On `main` branch builds (see `.github/workflows/documentation.yml`), we push the built docs to -the `documentation` branch. Netlify automatically re-deploys the docs pages whenever there is a change to that branch. - -## Releases -Rasa has implemented robust policies governing version naming, as well as release pace for major, minor, and patch releases. - -The values for a given version number (MAJOR.MINOR.PATCH) are incremented as follows: -- MAJOR version for incompatible API changes or other breaking changes. -- MINOR version for functionality added in a backward compatible manner. -- PATCH version for backward compatible bug fixes. - -The following table describes the version types and their expected *release cadence*: - -| Version Type | Description | Target Cadence | -|--------------|-----------------------------------------------------------------------------------------------------------------------------------------------|-----------------| -| Major | For significant changes, or when any backward-incompatible changes are introduced to the API or data model. | Every 1 - 2 yrs | -| Minor | For when new backward-compatible functionality is introduced, a minor feature is introduced, or when a set of smaller features is rolled out. | +/- Quarterly | -| Patch | For backward-compatible bug fixes that fix incorrect behavior. | As needed | - -While this table represents our target release frequency, we reserve the right to modify it based on changing market conditions and technical requirements. - -### Maintenance Policy -Our End of Life policy defines how long a given release is considered supported, as well as how long a release is -considered to be still in active development or maintenance. - -The maintentance duration and end of life for every release are shown on our website as part of the [Product Release and Maintenance Policy](https://rasa.com/rasa-product-release-and-maintenance-policy/). - -### Cutting a Major / Minor release -#### A week before release day - -1. **Make sure the [milestone](https://github.com/RasaHQ/rasa/milestones) already exists and is scheduled for the -correct date.** -2. **Take a look at the issues & PRs that are in the milestone**: does it look about right for the release highlights -we are planning to ship? Does it look like anything is missing? Don't worry about being aware of every PR that should -be in, but it's useful to take a moment to evaluate what's assigned to the milestone. -3. **Post a message on the engineering Slack channel**, letting the team know you'll be the one cutting the upcoming -release, as well as: - 1. Providing the link to the appropriate milestone - 2. Reminding everyone to go over their issues and PRs and please assign them to the milestone - 3. Reminding everyone of the scheduled date for the release - -#### A day before release day - -1. **Go over the milestone and evaluate the status of any PR merging that's happening. Follow up with people on their -bugs and fixes.** If the release introduces new bugs or regressions that can't be fixed in time, we should discuss on -Slack about this and take a decision on how to move forward. If the issue is not ready to be merged in time, we remove the issue / PR from the milestone and notify the PR owner and the product manager on Slack about it. The PR / issue owners are responsible for -communicating any issues which might be release relevant. Postponing the release should be considered as an edge case scenario. - -#### Release day! ๐Ÿš€ - -1. **At the start of the day, post a small message on slack announcing release day!** Communicate you'll be handling -the release, and the time you're aiming to start releasing (again, no later than 4pm, as issues may arise and -cause delays). This message should be posted early in the morning and before moving forward with any of the steps of the release, - in order to give enough time to people to check their PRs and issues. That way they can plan any remaining work. A template of the slack message can be found [here](https://rasa-hq.slack.com/archives/C36SS4N8M/p1613032208137500?thread_ts=1612876410.068400&cid=C36SS4N8M). - The release time should be communicated transparently so that others can plan potentially necessary steps accordingly. If there are bigger changes this should be communicated. -2. Make sure the milestone is empty (everything has been either merged or moved to the next milestone) -3. Once everything in the milestone is taken care of, post a small message on Slack communicating you are about to -start the release process (in case anything is missing). -4. **You may now do the release by following the instructions outlined in the -[Rasa Open Source README](#steps-to-release-a-new-version) !** - -#### After a Major release - -After a Major release has been completed, please follow [these instructions to complete the documentation update](./docs/README.md#manual-steps-after-a-new-version). - -### Steps to release a new version -Releasing a new version is quite simple, as the packages are build and distributed by GitHub Actions. - -*Release steps*: -1. Make sure all dependencies are up to date (**especially Rasa SDK**) - - For Rasa SDK, except in the case of a patch release, that means first creating a [new Rasa SDK release](https://github.com/RasaHQ/rasa-sdk#steps-to-release-a-new-version) (make sure the version numbers between the new Rasa and Rasa SDK releases match) - - Once the tag with the new Rasa SDK release is pushed and the package appears on [pypi](https://pypi.org/project/rasa-sdk/), the dependency in the rasa repository can be resolved (see below). -2. If this is a minor / major release: Make sure all fixes from currently supported minor versions have been merged from their respective release branches (e.g. 3.3.x) back into main. -3. In case of a minor release, create a new branch that corresponds to the new release, e.g. - ```bash - git checkout -b 1.2.x - git push origin 1.2.x - ``` -4. Switch to the branch you want to cut the release from (`main` in case of a major, the `..x` branch for minors and patches) - - Update the `rasa-sdk` entry in `pyproject.toml` with the new release version and run `poetry update`. This creates a new `poetry.lock` file with all dependencies resolved. - - Commit the changes with `git commit -am "bump rasa-sdk dependency"` but do not push them. They will be automatically picked up by the following step. -5. If this is a major release, update the list of actively maintained versions [in the README](#actively-maintained-versions) and in [the docs](./docs/docs/actively-maintained-versions.mdx). -6. Run `make release` -7. Create a PR against the release branch (e.g. `1.2.x`) -8. Once your PR is merged, tag a new release (this SHOULD always happen on the release branch), e.g. using +1. **Install Poetry**: Follow the [official guide](https://python-poetry.org/docs/#installation). +2. **Build from source**: ```bash - git checkout 1.2.x - git pull origin 1.2.x - git tag 1.2.0 -m "next release" - git push origin 1.2.0 --tags + make install ``` - GitHub will build this tag and publish the build artifacts. -9. After all the steps are completed and if everything goes well then we should see a message automatically posted in the company's Slack (`product` channel) like this [one](https://rasa-hq.slack.com/archives/C7B08Q5FX/p1614354499046600) -10. If no message appears in the channel then you can do the following checks: - - Check the workflows in [Github Actions](https://github.com/RasaHQ/rasa/actions) and make sure that the merged PR of the current release is completed successfully. To easily find your PR you can use the filters `event: push` and `branch: ` (example on release 2.4 you can see [here](https://github.com/RasaHQ/rasa/actions/runs/643344876)) - - If the workflow is not completed, then try to re run the workflow in case that solves the problem - - If the problem persists, check also the log files and try to find the root cause of the issue - - If you still cannot resolve the error, contact the infrastructure team by providing any helpful information from your investigation -11. After the message is posted correctly in the `product` channel, check also in the `product-engineering-alerts` channel if there are any alerts related to the Rasa Open Source release like this [one](https://rasa-hq.slack.com/archives/C01585AN2NP/p1615486087001000) + *Note for macOS users*: If you run into compiler issues, try `export SYSTEM_VERSION_COMPAT=1` before installation. -### Cutting a Patch release +### Running Tests +Make sure you have development requirements installed: -Patch releases are simpler to cut, since they are meant to contain only bugfixes. - -**The only things you need to do to cut a patch release are:** - -1. Notify the engineering team on Slack that you are planning to cut a patch, in case someone has an important fix -to add. -2. Make sure the bugfix(es) are in the release branch you will use (p.e if you are cutting a `2.0.4` patch, you will -need your fixes to be on the `2.0.x` release branch). All patch releases must come from a `.x` branch! -3. Once you're ready to release the Rasa Open Source patch, checkout the branch, run `make release` and follow the -steps + get the PR merged. -4. Once the PR is in, pull the `.x` branch again and push the tag! +```bash +make prepare-tests-ubuntu # Ubuntu/Debian +make prepare-tests-macos # macOS +make test # Run tests +``` -### Additional Release Tasks -**Note: This is only required if the released version is the highest version available. -For instance, perform the following steps when version > [version](https://github.com/RasaHQ/rasa/blob/main/rasa/version.py) on main.** +### Releases -In order to check compatibility between the new released Rasa version to the latest version of Rasa X/Enterprise, we perform the following steps: -1. Following a new Rasa release, an automated pull request is created in [Rasa-X-Demo](https://github.com/RasaHQ/rasa-x-demo/pulls). -2. Once the above PR is merged, follow instructions [here](https://github.com/RasaHQ/rasa-x-demo/blob/master/.github/VERSION_BUMPER_PR_COMMENT.md), to release a version. -3. Update the new version in the Rasa X/Enterprise [env file](https://github.com/RasaHQ/rasa-x/blob/main/.env). -The [Rasa-X-Demo](https://github.com/RasaHQ/rasa-x-demo) project uses the new updated Rasa version to train and test a model which in turn is used by our CI to run tests in the Rasa X/Enterprise repository, -thus validating compatibility between Rasa and Rasa X/Enterprise. +Rasa follows Semantic Versioning. -### Actively maintained versions + * **Major**: Incompatible API changes + * **Minor**: Backward-compatible functionality + * **Patch**: Backward-compatible bug fixes -Please refer to the [Rasa Product Release and Maintenance Policy](https://rasa.com/rasa-product-release-and-maintenance-policy/) page. +For full details on our release cadence and maintenance policy, visit our [Product Release and Maintenance Policy](https://rasa.com/rasa-product-release-and-maintenance-policy/). ## License -Licensed under the Apache License, Version 2.0. -Copyright 2022 Rasa Technologies GmbH. [Copy of the license](LICENSE.txt). -A list of the Licenses of the dependencies of the project can be found at -the bottom of the -[Libraries Summary](https://libraries.io/github/RasaHQ/rasa). +Licensed under the Apache License, Version 2.0. Copyright 2022 Rasa Technologies GmbH. [Copy of the license](https://www.google.com/search?q=LICENSE.txt). diff --git a/docker/docker-bake.hcl b/docker/docker-bake.hcl index 00e1471d0804..89934a81accf 100644 --- a/docker/docker-bake.hcl +++ b/docker/docker-bake.hcl @@ -21,7 +21,7 @@ variable "BASE_BUILDER_IMAGE_HASH" { # keep this in sync with the version in .github/poetry_version.txt # the variable is set automatically for builds in CI variable "POETRY_VERSION" { - default = "1.4.2" + default = "1.8.2" } group "base-images" { diff --git a/docs/docs/docker/building-in-docker.mdx b/docs/docs/docker/building-in-docker.mdx index a6027c51f7e2..9866ad95e063 100644 --- a/docs/docs/docker/building-in-docker.mdx +++ b/docs/docs/docker/building-in-docker.mdx @@ -67,7 +67,7 @@ The initial project files should all be there, as well as a `models` directory t :::note If you run into permission errors, it may be because the `rasa/rasa` images run as user `1001` as a best practice, to avoid giving the container `root` permissions. -Hence, all files created by these containers will be owned by user `1001`. See the [Docker documentation](https://docs.docker.com/edge/engine/reference/commandline/run/) +Hence, all files created by these containers will be owned by user `1001`. See the [Docker documentation](https://docs.docker.com/reference/cli/docker/container/run/) if you want to run the containers as a different user. ::: diff --git a/docs/docs/monitoring/load-testing-guidelines.mdx b/docs/docs/monitoring/load-testing-guidelines.mdx index ff40486853b5..a794d73639da 100644 --- a/docs/docs/monitoring/load-testing-guidelines.mdx +++ b/docs/docs/monitoring/load-testing-guidelines.mdx @@ -12,12 +12,26 @@ In order to gather metrics on our system's ability to handle increased loads and In each test case we spawned the following number of concurrent users at peak concurrency using a [spawn rate](https://docs.locust.io/en/1.5.0/configuration.html#all-available-configuration-options) of 1000 users per second. In our tests we used the Rasa [HTTP-API](https://rasa.com/docs/rasa/pages/http-api) and the [Locust](https://locust.io/) open source load testing tool. + | Users | CPU | Memory | |--------------------------|----------------------------------------------|---------------| | Up to 50,000 | 6vCPU | 16 GB | | Up to 80,000 | 6vCPU, with almost 90% CPU usage | 16 GB | +### Some recommendations to improve latency +- Sanic Workers must be mapped 1:1 to CPU for both Rasa Pro and Rasa Action Server +- Create `async` actions to avoid any blocking I/O +- `enable_selective_domain: true` : Domain is only sent for actions that needs it. This massively trims the payload between the two pods. +- Consider using compute efficient machines on cloud which are optimized for high performance computing such as the C5 instances on AWS. + However, as they are low on memory, models need to be trained lightweight. + + +| Machine | RasaPro | Rasa Action Server | +|--------------------------------|------------------------------------------------|--------------------------------------------------| +| AWS C5 or Azure F or Gcloud C2 | 3-7vCPU, 10-16Gb Memory, 3-7 Sanic Threads | 3-7vCPU, 2-12Gb Memory, 3-7 Sanic Threads | + + ### Debugging bot related issues while scaling up To test the Rasa [HTTP-API](https://rasa.com/docs/rasa/pages/http-api) ability to handle a large number of concurrent user activity we used the Rasa Pro [tracing](./tracing.mdx) capability diff --git a/poetry.lock b/poetry.lock index b347db8c13bf..ba2092a01ff4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,9 @@ -# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "absl-py" version = "1.4.0" description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -16,7 +15,6 @@ files = [ name = "aio-pika" version = "8.2.3" description = "Wrapper for the aiormq for asyncio and humans." -category = "main" optional = false python-versions = ">3.6, <4" files = [ @@ -35,7 +33,6 @@ develop = ["aiomisc (>=16.0,<17.0)", "coverage (!=4.3)", "coveralls", "nox", "py name = "aiofiles" version = "23.1.0" description = "File support for asyncio." -category = "main" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -45,140 +42,124 @@ files = [ [[package]] name = "aiogram" -version = "2.25.1" +version = "2.15" description = "Is a pretty simple and fully asynchronous framework for Telegram Bot API" -category = "main" optional = false -python-versions = ">=3.7" +python-versions = "*" files = [ - {file = "aiogram-2.25.1-py3-none-any.whl", hash = "sha256:7bb770cd0459f1dbaea00578bf13fb2e6a1812f22adf94a988c11a7c0d5f33e1"}, - {file = "aiogram-2.25.1.tar.gz", hash = "sha256:59ad78fc0ebbef1fd471c15778a4594b60117e0d7373bc2ce7bcd192074d527d"}, + {file = "aiogram-2.15-py3-none-any.whl", hash = "sha256:5d4dae610625893fe53e07c01c9e95671fd863718caab692baac948e3746ab87"}, + {file = "aiogram-2.15.tar.gz", hash = "sha256:13c740c52ee1301af8a9905e0a412754bcff03deb83dfdf1c578d9249ab35026"}, ] [package.dependencies] -aiohttp = ">=3.8.0,<3.9.0" -Babel = ">=2.9.1,<2.10.0" -certifi = ">=2021.10.8" -magic-filter = ">=1.0.9" +aiohttp = ">=3.7.2,<4.0.0" +Babel = ">=2.8.0" +certifi = ">=2020.6.20" [package.extras] -fast = ["ujson (>=1.35)", "uvloop (>=0.16.0,<0.17.0)"] +fast = ["ujson (>=1.35)", "uvloop (>=0.14.0,<0.15.0)"] proxy = ["aiohttp-socks (>=0.5.3,<0.6.0)"] [[package]] name = "aiohttp" -version = "3.8.4" +version = "3.9.3" description = "Async http client/server framework (asyncio)" -category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "aiohttp-3.8.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5ce45967538fb747370308d3145aa68a074bdecb4f3a300869590f725ced69c1"}, - {file = "aiohttp-3.8.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b744c33b6f14ca26b7544e8d8aadff6b765a80ad6164fb1a430bbadd593dfb1a"}, - {file = "aiohttp-3.8.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1a45865451439eb320784918617ba54b7a377e3501fb70402ab84d38c2cd891b"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a86d42d7cba1cec432d47ab13b6637bee393a10f664c425ea7b305d1301ca1a3"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee3c36df21b5714d49fc4580247947aa64bcbe2939d1b77b4c8dcb8f6c9faecc"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:176a64b24c0935869d5bbc4c96e82f89f643bcdf08ec947701b9dbb3c956b7dd"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c844fd628851c0bc309f3c801b3a3d58ce430b2ce5b359cd918a5a76d0b20cb5"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5393fb786a9e23e4799fec788e7e735de18052f83682ce2dfcabaf1c00c2c08e"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e4b09863aae0dc965c3ef36500d891a3ff495a2ea9ae9171e4519963c12ceefd"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:adfbc22e87365a6e564c804c58fc44ff7727deea782d175c33602737b7feadb6"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:147ae376f14b55f4f3c2b118b95be50a369b89b38a971e80a17c3fd623f280c9"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:eafb3e874816ebe2a92f5e155f17260034c8c341dad1df25672fb710627c6949"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c6cc15d58053c76eacac5fa9152d7d84b8d67b3fde92709195cb984cfb3475ea"}, - {file = "aiohttp-3.8.4-cp310-cp310-win32.whl", hash = "sha256:59f029a5f6e2d679296db7bee982bb3d20c088e52a2977e3175faf31d6fb75d1"}, - {file = "aiohttp-3.8.4-cp310-cp310-win_amd64.whl", hash = "sha256:fe7ba4a51f33ab275515f66b0a236bcde4fb5561498fe8f898d4e549b2e4509f"}, - {file = "aiohttp-3.8.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3d8ef1a630519a26d6760bc695842579cb09e373c5f227a21b67dc3eb16cfea4"}, - {file = "aiohttp-3.8.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b3f2e06a512e94722886c0827bee9807c86a9f698fac6b3aee841fab49bbfb4"}, - {file = "aiohttp-3.8.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a80464982d41b1fbfe3154e440ba4904b71c1a53e9cd584098cd41efdb188ef"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b631e26df63e52f7cce0cce6507b7a7f1bc9b0c501fcde69742130b32e8782f"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f43255086fe25e36fd5ed8f2ee47477408a73ef00e804cb2b5cba4bf2ac7f5e"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4d347a172f866cd1d93126d9b239fcbe682acb39b48ee0873c73c933dd23bd0f"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3fec6a4cb5551721cdd70473eb009d90935b4063acc5f40905d40ecfea23e05"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80a37fe8f7c1e6ce8f2d9c411676e4bc633a8462844e38f46156d07a7d401654"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d1e6a862b76f34395a985b3cd39a0d949ca80a70b6ebdea37d3ab39ceea6698a"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cd468460eefef601ece4428d3cf4562459157c0f6523db89365202c31b6daebb"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:618c901dd3aad4ace71dfa0f5e82e88b46ef57e3239fc7027773cb6d4ed53531"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:652b1bff4f15f6287550b4670546a2947f2a4575b6c6dff7760eafb22eacbf0b"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80575ba9377c5171407a06d0196b2310b679dc752d02a1fcaa2bc20b235dbf24"}, - {file = "aiohttp-3.8.4-cp311-cp311-win32.whl", hash = "sha256:bbcf1a76cf6f6dacf2c7f4d2ebd411438c275faa1dc0c68e46eb84eebd05dd7d"}, - {file = "aiohttp-3.8.4-cp311-cp311-win_amd64.whl", hash = "sha256:6e74dd54f7239fcffe07913ff8b964e28b712f09846e20de78676ce2a3dc0bfc"}, - {file = "aiohttp-3.8.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:880e15bb6dad90549b43f796b391cfffd7af373f4646784795e20d92606b7a51"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb96fa6b56bb536c42d6a4a87dfca570ff8e52de2d63cabebfd6fb67049c34b6"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a6cadebe132e90cefa77e45f2d2f1a4b2ce5c6b1bfc1656c1ddafcfe4ba8131"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f352b62b45dff37b55ddd7b9c0c8672c4dd2eb9c0f9c11d395075a84e2c40f75"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ab43061a0c81198d88f39aaf90dae9a7744620978f7ef3e3708339b8ed2ef01"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9cb1565a7ad52e096a6988e2ee0397f72fe056dadf75d17fa6b5aebaea05622"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:1b3ea7edd2d24538959c1c1abf97c744d879d4e541d38305f9bd7d9b10c9ec41"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:7c7837fe8037e96b6dd5cfcf47263c1620a9d332a87ec06a6ca4564e56bd0f36"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:3b90467ebc3d9fa5b0f9b6489dfb2c304a1db7b9946fa92aa76a831b9d587e99"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:cab9401de3ea52b4b4c6971db5fb5c999bd4260898af972bf23de1c6b5dd9d71"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:d1f9282c5f2b5e241034a009779e7b2a1aa045f667ff521e7948ea9b56e0c5ff"}, - {file = "aiohttp-3.8.4-cp36-cp36m-win32.whl", hash = "sha256:5e14f25765a578a0a634d5f0cd1e2c3f53964553a00347998dfdf96b8137f777"}, - {file = "aiohttp-3.8.4-cp36-cp36m-win_amd64.whl", hash = "sha256:4c745b109057e7e5f1848c689ee4fb3a016c8d4d92da52b312f8a509f83aa05e"}, - {file = "aiohttp-3.8.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:aede4df4eeb926c8fa70de46c340a1bc2c6079e1c40ccf7b0eae1313ffd33519"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ddaae3f3d32fc2cb4c53fab020b69a05c8ab1f02e0e59665c6f7a0d3a5be54f"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4eb3b82ca349cf6fadcdc7abcc8b3a50ab74a62e9113ab7a8ebc268aad35bb9"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9bcb89336efa095ea21b30f9e686763f2be4478f1b0a616969551982c4ee4c3b"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c08e8ed6fa3d477e501ec9db169bfac8140e830aa372d77e4a43084d8dd91ab"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c6cd05ea06daca6ad6a4ca3ba7fe7dc5b5de063ff4daec6170ec0f9979f6c332"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7a00a9ed8d6e725b55ef98b1b35c88013245f35f68b1b12c5cd4100dddac333"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:de04b491d0e5007ee1b63a309956eaed959a49f5bb4e84b26c8f5d49de140fa9"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:40653609b3bf50611356e6b6554e3a331f6879fa7116f3959b20e3528783e699"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dbf3a08a06b3f433013c143ebd72c15cac33d2914b8ea4bea7ac2c23578815d6"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854f422ac44af92bfe172d8e73229c270dc09b96535e8a548f99c84f82dde241"}, - {file = "aiohttp-3.8.4-cp37-cp37m-win32.whl", hash = "sha256:aeb29c84bb53a84b1a81c6c09d24cf33bb8432cc5c39979021cc0f98c1292a1a"}, - {file = "aiohttp-3.8.4-cp37-cp37m-win_amd64.whl", hash = "sha256:db3fc6120bce9f446d13b1b834ea5b15341ca9ff3f335e4a951a6ead31105480"}, - {file = "aiohttp-3.8.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:fabb87dd8850ef0f7fe2b366d44b77d7e6fa2ea87861ab3844da99291e81e60f"}, - {file = "aiohttp-3.8.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91f6d540163f90bbaef9387e65f18f73ffd7c79f5225ac3d3f61df7b0d01ad15"}, - {file = "aiohttp-3.8.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d265f09a75a79a788237d7f9054f929ced2e69eb0bb79de3798c468d8a90f945"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d89efa095ca7d442a6d0cbc755f9e08190ba40069b235c9886a8763b03785da"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4dac314662f4e2aa5009977b652d9b8db7121b46c38f2073bfeed9f4049732cd"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe11310ae1e4cd560035598c3f29d86cef39a83d244c7466f95c27ae04850f10"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ddb2a2026c3f6a68c3998a6c47ab6795e4127315d2e35a09997da21865757f8"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e75b89ac3bd27d2d043b234aa7b734c38ba1b0e43f07787130a0ecac1e12228a"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6e601588f2b502c93c30cd5a45bfc665faaf37bbe835b7cfd461753068232074"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a5d794d1ae64e7753e405ba58e08fcfa73e3fad93ef9b7e31112ef3c9a0efb52"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:a1f4689c9a1462f3df0a1f7e797791cd6b124ddbee2b570d34e7f38ade0e2c71"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:3032dcb1c35bc330134a5b8a5d4f68c1a87252dfc6e1262c65a7e30e62298275"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8189c56eb0ddbb95bfadb8f60ea1b22fcfa659396ea36f6adcc521213cd7b44d"}, - {file = "aiohttp-3.8.4-cp38-cp38-win32.whl", hash = "sha256:33587f26dcee66efb2fff3c177547bd0449ab7edf1b73a7f5dea1e38609a0c54"}, - {file = "aiohttp-3.8.4-cp38-cp38-win_amd64.whl", hash = "sha256:e595432ac259af2d4630008bf638873d69346372d38255774c0e286951e8b79f"}, - {file = "aiohttp-3.8.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5a7bdf9e57126dc345b683c3632e8ba317c31d2a41acd5800c10640387d193ed"}, - {file = "aiohttp-3.8.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:22f6eab15b6db242499a16de87939a342f5a950ad0abaf1532038e2ce7d31567"}, - {file = "aiohttp-3.8.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7235604476a76ef249bd64cb8274ed24ccf6995c4a8b51a237005ee7a57e8643"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea9eb976ffdd79d0e893869cfe179a8f60f152d42cb64622fca418cd9b18dc2a"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92c0cea74a2a81c4c76b62ea1cac163ecb20fb3ba3a75c909b9fa71b4ad493cf"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:493f5bc2f8307286b7799c6d899d388bbaa7dfa6c4caf4f97ef7521b9cb13719"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a63f03189a6fa7c900226e3ef5ba4d3bd047e18f445e69adbd65af433add5a2"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10c8cefcff98fd9168cdd86c4da8b84baaa90bf2da2269c6161984e6737bf23e"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bca5f24726e2919de94f047739d0a4fc01372801a3672708260546aa2601bf57"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:03baa76b730e4e15a45f81dfe29a8d910314143414e528737f8589ec60cf7391"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:8c29c77cc57e40f84acef9bfb904373a4e89a4e8b74e71aa8075c021ec9078c2"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:03543dcf98a6619254b409be2d22b51f21ec66272be4ebda7b04e6412e4b2e14"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:17b79c2963db82086229012cff93ea55196ed31f6493bb1ccd2c62f1724324e4"}, - {file = "aiohttp-3.8.4-cp39-cp39-win32.whl", hash = "sha256:34ce9f93a4a68d1272d26030655dd1b58ff727b3ed2a33d80ec433561b03d67a"}, - {file = "aiohttp-3.8.4-cp39-cp39-win_amd64.whl", hash = "sha256:41a86a69bb63bb2fc3dc9ad5ea9f10f1c9c8e282b471931be0268ddd09430b04"}, - {file = "aiohttp-3.8.4.tar.gz", hash = "sha256:bf2e1a9162c1e441bf805a1fd166e249d574ca04e03b34f97e2928769e91ab5c"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:939677b61f9d72a4fa2a042a5eee2a99a24001a67c13da113b2e30396567db54"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1f5cd333fcf7590a18334c90f8c9147c837a6ec8a178e88d90a9b96ea03194cc"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:82e6aa28dd46374f72093eda8bcd142f7771ee1eb9d1e223ff0fa7177a96b4a5"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f56455b0c2c7cc3b0c584815264461d07b177f903a04481dfc33e08a89f0c26b"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bca77a198bb6e69795ef2f09a5f4c12758487f83f33d63acde5f0d4919815768"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e083c285857b78ee21a96ba1eb1b5339733c3563f72980728ca2b08b53826ca5"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab40e6251c3873d86ea9b30a1ac6d7478c09277b32e14745d0d3c6e76e3c7e29"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:df822ee7feaaeffb99c1a9e5e608800bd8eda6e5f18f5cfb0dc7eeb2eaa6bbec"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:acef0899fea7492145d2bbaaaec7b345c87753168589cc7faf0afec9afe9b747"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cd73265a9e5ea618014802ab01babf1940cecb90c9762d8b9e7d2cc1e1969ec6"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:a78ed8a53a1221393d9637c01870248a6f4ea5b214a59a92a36f18151739452c"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:6b0e029353361f1746bac2e4cc19b32f972ec03f0f943b390c4ab3371840aabf"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7cf5c9458e1e90e3c390c2639f1017a0379a99a94fdfad3a1fd966a2874bba52"}, + {file = "aiohttp-3.9.3-cp310-cp310-win32.whl", hash = "sha256:3e59c23c52765951b69ec45ddbbc9403a8761ee6f57253250c6e1536cacc758b"}, + {file = "aiohttp-3.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:055ce4f74b82551678291473f66dc9fb9048a50d8324278751926ff0ae7715e5"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6b88f9386ff1ad91ace19d2a1c0225896e28815ee09fc6a8932fded8cda97c3d"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c46956ed82961e31557b6857a5ca153c67e5476972e5f7190015018760938da2"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:07b837ef0d2f252f96009e9b8435ec1fef68ef8b1461933253d318748ec1acdc"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad46e6f620574b3b4801c68255492e0159d1712271cc99d8bdf35f2043ec266"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ed3e046ea7b14938112ccd53d91c1539af3e6679b222f9469981e3dac7ba1ce"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:039df344b45ae0b34ac885ab5b53940b174530d4dd8a14ed8b0e2155b9dddccb"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7943c414d3a8d9235f5f15c22ace69787c140c80b718dcd57caaade95f7cd93b"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84871a243359bb42c12728f04d181a389718710129b36b6aad0fc4655a7647d4"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5eafe2c065df5401ba06821b9a054d9cb2848867f3c59801b5d07a0be3a380ae"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9d3c9b50f19704552f23b4eaea1fc082fdd82c63429a6506446cbd8737823da3"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:f033d80bc6283092613882dfe40419c6a6a1527e04fc69350e87a9df02bbc283"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:2c895a656dd7e061b2fd6bb77d971cc38f2afc277229ce7dd3552de8313a483e"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1f5a71d25cd8106eab05f8704cd9167b6e5187bcdf8f090a66c6d88b634802b4"}, + {file = "aiohttp-3.9.3-cp311-cp311-win32.whl", hash = "sha256:50fca156d718f8ced687a373f9e140c1bb765ca16e3d6f4fe116e3df7c05b2c5"}, + {file = "aiohttp-3.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:5fe9ce6c09668063b8447f85d43b8d1c4e5d3d7e92c63173e6180b2ac5d46dd8"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:38a19bc3b686ad55804ae931012f78f7a534cce165d089a2059f658f6c91fa60"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:770d015888c2a598b377bd2f663adfd947d78c0124cfe7b959e1ef39f5b13869"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ee43080e75fc92bf36219926c8e6de497f9b247301bbf88c5c7593d931426679"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52df73f14ed99cee84865b95a3d9e044f226320a87af208f068ecc33e0c35b96"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc9b311743a78043b26ffaeeb9715dc360335e5517832f5a8e339f8a43581e4d"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b955ed993491f1a5da7f92e98d5dad3c1e14dc175f74517c4e610b1f2456fb11"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:504b6981675ace64c28bf4a05a508af5cde526e36492c98916127f5a02354d53"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a6fe5571784af92b6bc2fda8d1925cccdf24642d49546d3144948a6a1ed58ca5"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ba39e9c8627edc56544c8628cc180d88605df3892beeb2b94c9bc857774848ca"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e5e46b578c0e9db71d04c4b506a2121c0cb371dd89af17a0586ff6769d4c58c1"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:938a9653e1e0c592053f815f7028e41a3062e902095e5a7dc84617c87267ebd5"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:c3452ea726c76e92f3b9fae4b34a151981a9ec0a4847a627c43d71a15ac32aa6"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ff30218887e62209942f91ac1be902cc80cddb86bf00fbc6783b7a43b2bea26f"}, + {file = "aiohttp-3.9.3-cp312-cp312-win32.whl", hash = "sha256:38f307b41e0bea3294a9a2a87833191e4bcf89bb0365e83a8be3a58b31fb7f38"}, + {file = "aiohttp-3.9.3-cp312-cp312-win_amd64.whl", hash = "sha256:b791a3143681a520c0a17e26ae7465f1b6f99461a28019d1a2f425236e6eedb5"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0ed621426d961df79aa3b963ac7af0d40392956ffa9be022024cd16297b30c8c"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7f46acd6a194287b7e41e87957bfe2ad1ad88318d447caf5b090012f2c5bb528"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:feeb18a801aacb098220e2c3eea59a512362eb408d4afd0c242044c33ad6d542"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f734e38fd8666f53da904c52a23ce517f1b07722118d750405af7e4123933511"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b40670ec7e2156d8e57f70aec34a7216407848dfe6c693ef131ddf6e76feb672"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fdd215b7b7fd4a53994f238d0f46b7ba4ac4c0adb12452beee724ddd0743ae5d"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:017a21b0df49039c8f46ca0971b3a7fdc1f56741ab1240cb90ca408049766168"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e99abf0bba688259a496f966211c49a514e65afa9b3073a1fcee08856e04425b"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:648056db9a9fa565d3fa851880f99f45e3f9a771dd3ff3bb0c048ea83fb28194"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8aacb477dc26797ee089721536a292a664846489c49d3ef9725f992449eda5a8"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:522a11c934ea660ff8953eda090dcd2154d367dec1ae3c540aff9f8a5c109ab4"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:5bce0dc147ca85caa5d33debc4f4d65e8e8b5c97c7f9f660f215fa74fc49a321"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b4af9f25b49a7be47c0972139e59ec0e8285c371049df1a63b6ca81fdd216a2"}, + {file = "aiohttp-3.9.3-cp38-cp38-win32.whl", hash = "sha256:298abd678033b8571995650ccee753d9458dfa0377be4dba91e4491da3f2be63"}, + {file = "aiohttp-3.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:69361bfdca5468c0488d7017b9b1e5ce769d40b46a9f4a2eed26b78619e9396c"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0fa43c32d1643f518491d9d3a730f85f5bbaedcbd7fbcae27435bb8b7a061b29"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:835a55b7ca49468aaaac0b217092dfdff370e6c215c9224c52f30daaa735c1c1"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06a9b2c8837d9a94fae16c6223acc14b4dfdff216ab9b7202e07a9a09541168f"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abf151955990d23f84205286938796c55ff11bbfb4ccfada8c9c83ae6b3c89a3"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59c26c95975f26e662ca78fdf543d4eeaef70e533a672b4113dd888bd2423caa"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f95511dd5d0e05fd9728bac4096319f80615aaef4acbecb35a990afebe953b0e"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:595f105710293e76b9dc09f52e0dd896bd064a79346234b521f6b968ffdd8e58"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7c8b816c2b5af5c8a436df44ca08258fc1a13b449393a91484225fcb7545533"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f1088fa100bf46e7b398ffd9904f4808a0612e1d966b4aa43baa535d1b6341eb"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f59dfe57bb1ec82ac0698ebfcdb7bcd0e99c255bd637ff613760d5f33e7c81b3"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:361a1026c9dd4aba0109e4040e2aecf9884f5cfe1b1b1bd3d09419c205e2e53d"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:363afe77cfcbe3a36353d8ea133e904b108feea505aa4792dad6585a8192c55a"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e2c45c208c62e955e8256949eb225bd8b66a4c9b6865729a786f2aa79b72e9d"}, + {file = "aiohttp-3.9.3-cp39-cp39-win32.whl", hash = "sha256:f7217af2e14da0856e082e96ff637f14ae45c10a5714b63c77f26d8884cf1051"}, + {file = "aiohttp-3.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:27468897f628c627230dba07ec65dc8d0db566923c48f29e084ce382119802bc"}, + {file = "aiohttp-3.9.3.tar.gz", hash = "sha256:90842933e5d1ff760fae6caca4b2b3edba53ba8f4b71e95dacf2818a2aca06f7"}, ] [package.dependencies] aiosignal = ">=1.1.2" -async-timeout = ">=4.0.0a3,<5.0" +async-timeout = {version = ">=4.0,<5.0", markers = "python_version < \"3.11\""} attrs = ">=17.3.0" -charset-normalizer = ">=2.0,<4.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" yarl = ">=1.0,<2.0" [package.extras] -speedups = ["Brotli", "aiodns", "cchardet"] +speedups = ["Brotli", "aiodns", "brotlicffi"] [[package]] name = "aiohttp-retry" version = "2.8.3" description = "Simple retry client for aiohttp" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -191,24 +172,22 @@ aiohttp = "*" [[package]] name = "aioresponses" -version = "0.7.4" +version = "0.7.6" description = "Mock out requests made by ClientSession from aiohttp package" -category = "dev" optional = false python-versions = "*" files = [ - {file = "aioresponses-0.7.4-py2.py3-none-any.whl", hash = "sha256:1160486b5ea96fcae6170cf2bdef029b9d3a283b7dbeabb3d7f1182769bfb6b7"}, - {file = "aioresponses-0.7.4.tar.gz", hash = "sha256:9b8c108b36354c04633bad0ea752b55d956a7602fe3e3234b939fc44af96f1d8"}, + {file = "aioresponses-0.7.6-py2.py3-none-any.whl", hash = "sha256:d2c26defbb9b440ea2685ec132e90700907fd10bcca3e85ec2f157219f0d26f7"}, + {file = "aioresponses-0.7.6.tar.gz", hash = "sha256:f795d9dbda2d61774840e7e32f5366f45752d1adc1b74c9362afd017296c7ee1"}, ] [package.dependencies] -aiohttp = ">=2.0.0,<4.0.0" +aiohttp = ">=3.3.0,<4.0.0" [[package]] name = "aiormq" version = "6.4.2" description = "Pure python AMQP asynchronous client library" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -227,7 +206,6 @@ develop = ["aiomisc (>=16.0,<17.0)", "coverage (!=4.3)", "coveralls", "pylava", name = "aiosignal" version = "1.3.1" description = "aiosignal: a list of registered asynchronous callbacks" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -242,7 +220,6 @@ frozenlist = ">=1.1.0" name = "analytics-python" version = "1.4.post1" description = "The hassle-free way to integrate analytics into any python application." -category = "dev" optional = false python-versions = "*" files = [ @@ -264,7 +241,6 @@ test = ["flake8 (==3.7.9)", "mock (==2.0.0)", "pylint (==1.9.3)"] name = "anyio" version = "3.7.1" description = "High level compatibility layer for multiple asynchronous event loop implementations" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -286,7 +262,6 @@ trio = ["trio (<0.22)"] name = "apscheduler" version = "3.9.1.post1" description = "In-process task scheduler with Cron-like capabilities" -category = "main" optional = false python-versions = "!=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" files = [ @@ -298,7 +273,7 @@ files = [ pytz = "*" setuptools = ">=0.7" six = ">=1.4.0" -tzlocal = ">=2.0,<3.0.0 || >=4.0.0" +tzlocal = ">=2.0,<3.dev0 || >=4.dev0" [package.extras] asyncio = ["trollius"] @@ -317,7 +292,6 @@ zookeeper = ["kazoo"] name = "astunparse" version = "1.6.3" description = "An AST unparser for Python" -category = "main" optional = false python-versions = "*" files = [ @@ -333,7 +307,6 @@ wheel = ">=0.23.0,<1.0" name = "async-generator" version = "1.10" description = "Async generators and context managers for Python 3.5+" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -345,7 +318,6 @@ files = [ name = "async-timeout" version = "4.0.2" description = "Timeout context manager for asyncio programs" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -357,7 +329,6 @@ files = [ name = "attrs" version = "22.1.0" description = "Classes Without Boilerplate" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -375,7 +346,6 @@ tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy name = "azure-core" version = "1.27.1" description = "Microsoft Azure Core Library for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -395,7 +365,6 @@ aio = ["aiohttp (>=3.0)"] name = "azure-storage-blob" version = "12.15.0" description = "Microsoft Azure Blob Storage Client Library for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -416,7 +385,6 @@ aio = ["azure-core[aio] (>=1.26.0,<2.0.0)"] name = "babel" version = "2.9.1" description = "Internationalization utilities" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -431,7 +399,6 @@ pytz = ">=2015.7" name = "backoff" version = "1.10.0" description = "Function decoration for backoff and retry" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -443,7 +410,6 @@ files = [ name = "backports-zoneinfo" version = "0.2.1" description = "Backport of the standard library zoneinfo module" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -472,7 +438,6 @@ tzdata = ["tzdata"] name = "bandit" version = "1.7.5" description = "Security oriented static analyser for python code." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -496,7 +461,6 @@ yaml = ["PyYAML"] name = "bidict" version = "0.22.1" description = "The bidirectional mapping library for Python." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -513,7 +477,6 @@ test = ["hypothesis", "pytest", "pytest-benchmark[histogram]", "pytest-cov", "py name = "black" version = "22.12.0" description = "The uncompromising code formatter." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -549,7 +512,6 @@ uvloop = ["uvloop (>=0.15.2)"] name = "blis" version = "0.7.9" description = "The Blis BLAS-like linear algebra library, as a self-contained C-extension." -category = "main" optional = true python-versions = "*" files = [ @@ -590,7 +552,6 @@ numpy = ">=1.15.0" name = "boto3" version = "1.27.1" description = "The AWS SDK for Python" -category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -610,7 +571,6 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] name = "botocore" version = "1.30.1" description = "Low-level, data-driven core of boto 3." -category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -630,7 +590,6 @@ crt = ["awscrt (==0.16.9)"] name = "cachecontrol" version = "0.12.14" description = "httplib2 caching for requests" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -650,7 +609,6 @@ redis = ["redis (>=2.10.5)"] name = "cachetools" version = "5.3.1" description = "Extensible memoizing collections and decorators" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -662,7 +620,6 @@ files = [ name = "catalogue" version = "2.0.8" description = "Super lightweight function registries for your library" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -674,7 +631,6 @@ files = [ name = "certifi" version = "2023.7.22" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -686,7 +642,6 @@ files = [ name = "cffi" version = "1.15.1" description = "Foreign Function Interface for Python calling C code." -category = "main" optional = false python-versions = "*" files = [ @@ -763,7 +718,6 @@ pycparser = "*" name = "charset-normalizer" version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -848,7 +802,6 @@ files = [ name = "click" version = "8.1.3" description = "Composable command line interface toolkit" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -863,7 +816,6 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "click-default-group" version = "1.2.2" description = "Extends click.Group to invoke a command without explicit subcommand name" -category = "dev" optional = false python-versions = "*" files = [ @@ -877,7 +829,6 @@ click = "*" name = "cloudpickle" version = "2.2.1" description = "Extended pickling support for Python objects" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -889,7 +840,6 @@ files = [ name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -901,7 +851,6 @@ files = [ name = "colorclass" version = "2.2.2" description = "Colorful worry-free console applications for Linux, Mac OS X, and Windows." -category = "main" optional = false python-versions = ">=2.6" files = [ @@ -913,7 +862,6 @@ files = [ name = "coloredlogs" version = "15.0.1" description = "Colored terminal output for Python's logging module" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -931,7 +879,6 @@ cron = ["capturer (>=2.4)"] name = "colorhash" version = "1.2.1" description = "Generate color based on any object" -category = "main" optional = false python-versions = ">=3.6,<4.0" files = [ @@ -943,7 +890,6 @@ files = [ name = "confection" version = "0.1.0" description = "The sweetest config system for Python" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -959,7 +905,6 @@ srsly = ">=2.4.0,<3.0.0" name = "confluent-kafka" version = "2.1.1" description = "Confluent's Python client for Apache Kafka" -category = "main" optional = false python-versions = "*" files = [ @@ -1006,7 +951,6 @@ schema-registry = ["requests"] name = "coverage" version = "6.5.0" description = "Code coverage measurement for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1072,7 +1016,6 @@ toml = ["tomli"] name = "coveralls" version = "3.3.1" description = "Show coverage stats online via coveralls.io" -category = "dev" optional = false python-versions = ">= 3.5" files = [ @@ -1081,7 +1024,7 @@ files = [ ] [package.dependencies] -coverage = ">=4.1,<6.0.0 || >6.1,<6.1.1 || >6.1.1,<7.0" +coverage = ">=4.1,<6.0.dev0 || >6.1,<6.1.1 || >6.1.1,<7.0" docopt = ">=0.6.1" requests = ">=1.0.0" @@ -1090,35 +1033,34 @@ yaml = ["PyYAML (>=3.10)"] [[package]] name = "cryptography" -version = "41.0.3" +version = "41.0.7" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-41.0.3-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:652627a055cb52a84f8c448185922241dd5217443ca194d5739b44612c5e6507"}, - {file = "cryptography-41.0.3-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:8f09daa483aedea50d249ef98ed500569841d6498aa9c9f4b0531b9964658922"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fd871184321100fb400d759ad0cddddf284c4b696568204d281c902fc7b0d81"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84537453d57f55a50a5b6835622ee405816999a7113267739a1b4581f83535bd"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3fb248989b6363906827284cd20cca63bb1a757e0a2864d4c1682a985e3dca47"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:42cb413e01a5d36da9929baa9d70ca90d90b969269e5a12d39c1e0d475010116"}, - {file = "cryptography-41.0.3-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:aeb57c421b34af8f9fe830e1955bf493a86a7996cc1338fe41b30047d16e962c"}, - {file = "cryptography-41.0.3-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:6af1c6387c531cd364b72c28daa29232162010d952ceb7e5ca8e2827526aceae"}, - {file = "cryptography-41.0.3-cp37-abi3-win32.whl", hash = "sha256:0d09fb5356f975974dbcb595ad2d178305e5050656affb7890a1583f5e02a306"}, - {file = "cryptography-41.0.3-cp37-abi3-win_amd64.whl", hash = "sha256:a983e441a00a9d57a4d7c91b3116a37ae602907a7618b882c8013b5762e80574"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5259cb659aa43005eb55a0e4ff2c825ca111a0da1814202c64d28a985d33b087"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:67e120e9a577c64fe1f611e53b30b3e69744e5910ff3b6e97e935aeb96005858"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:7efe8041897fe7a50863e51b77789b657a133c75c3b094e51b5e4b5cec7bf906"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ce785cf81a7bdade534297ef9e490ddff800d956625020ab2ec2780a556c313e"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:57a51b89f954f216a81c9d057bf1a24e2f36e764a1ca9a501a6964eb4a6800dd"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:4c2f0d35703d61002a2bbdcf15548ebb701cfdd83cdc12471d2bae80878a4207"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:23c2d778cf829f7d0ae180600b17e9fceea3c2ef8b31a99e3c694cbbf3a24b84"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:95dd7f261bb76948b52a5330ba5202b91a26fbac13ad0e9fc8a3ac04752058c7"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:41d7aa7cdfded09b3d73a47f429c298e80796c8e825ddfadc84c8a7f12df212d"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d0d651aa754ef58d75cec6edfbd21259d93810b73f6ec246436a21b7841908de"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ab8de0d091acbf778f74286f4989cf3d1528336af1b59f3e5d2ebca8b5fe49e1"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a74fbcdb2a0d46fe00504f571a2a540532f4c188e6ccf26f1f178480117b33c4"}, - {file = "cryptography-41.0.3.tar.gz", hash = "sha256:6d192741113ef5e30d89dcb5b956ef4e1578f304708701b8b73d38e3e1461f34"}, + {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf"}, + {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1"}, + {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157"}, + {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406"}, + {file = "cryptography-41.0.7-cp37-abi3-win32.whl", hash = "sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d"}, + {file = "cryptography-41.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309"}, + {file = "cryptography-41.0.7.tar.gz", hash = "sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc"}, ] [package.dependencies] @@ -1138,7 +1080,6 @@ test-randomorder = ["pytest-randomly"] name = "cycler" version = "0.11.0" description = "Composable style cycles" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1150,7 +1091,6 @@ files = [ name = "cymem" version = "2.0.7" description = "Manage calls to calloc/free through Cython" -category = "main" optional = true python-versions = "*" files = [ @@ -1188,7 +1128,6 @@ files = [ name = "dask" version = "2022.10.2" description = "Parallel PyData with Task Scheduling" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1217,7 +1156,6 @@ test = ["pandas[test]", "pre-commit", "pytest", "pytest-rerunfailures", "pytest- name = "databind" version = "1.5.3" description = "Databind is a library inspired by jackson-databind to de-/serialize Python dataclasses. The `databind` package will install the full suite of databind packages. Compatible with Python 3.7 and newer." -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -1233,7 +1171,6 @@ files = [ name = "databind-core" version = "1.5.3" description = "Databind is a library inspired by jackson-databind to de-/serialize Python dataclasses. Compatible with Python 3.7 and newer." -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -1250,7 +1187,6 @@ typing-extensions = ">=3.10.0" name = "databind-json" version = "1.5.3" description = "De-/serialize Python dataclasses to or from JSON payloads. Compatible with Python 3.7 and newer." -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -1267,7 +1203,6 @@ typing-extensions = ">=3.10.0" name = "datadog" version = "0.45.0" description = "The Datadog Python library" -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -1282,7 +1217,6 @@ requests = ">=2.6.0" name = "datadog-api-client" version = "2.14.0" description = "Collection of all Datadog Public endpoints" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1306,7 +1240,6 @@ zstandard = ["zstandard"] name = "deprecated" version = "1.2.14" description = "Python @deprecated decorator to deprecate old python classes, functions or methods." -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1324,7 +1257,6 @@ dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] name = "dnspython" version = "2.3.0" description = "DNS toolkit" -category = "main" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -1345,7 +1277,6 @@ wmi = ["wmi (>=1.5.1,<2.0.0)"] name = "docker" version = "6.1.3" description = "A Python library for the Docker Engine API." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1367,7 +1298,6 @@ ssh = ["paramiko (>=2.4.3)"] name = "docopt" version = "0.6.2" description = "Pythonic argument parser, that will make you smile" -category = "main" optional = false python-versions = "*" files = [ @@ -1378,7 +1308,6 @@ files = [ name = "docspec" version = "2.1.2" description = "Docspec is a JSON object specification for representing API documentation of programming languages." -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -1394,7 +1323,6 @@ Deprecated = ">=1.2.12,<2.0.0" name = "docspec-python" version = "2.0.2" description = "A parser based on lib2to3 producing docspec data from Python source code." -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -1410,7 +1338,6 @@ docspec = ">=2.0.2,<3.0.0" name = "docstring-parser" version = "0.11" description = "\"Parse Python docstrings in reST, Google and Numpydoc format\"" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1424,7 +1351,6 @@ test = ["black", "pytest"] name = "exceptiongroup" version = "1.1.2" description = "Backport of PEP 654 (exception groups)" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1439,7 +1365,6 @@ test = ["pytest (>=6)"] name = "execnet" version = "1.9.0" description = "execnet: rapid multi-Python deployment" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -1454,7 +1379,6 @@ testing = ["pre-commit"] name = "fakeredis" version = "2.16.0" description = "Python implementation of redis API, can be used for testing purposes." -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -1474,7 +1398,6 @@ lua = ["lupa (>=1.14,<2.0)"] name = "fbmessenger" version = "6.0.0" description = "A python library to communicate with the Facebook Messenger API's" -category = "main" optional = false python-versions = "*" files = [ @@ -1489,8 +1412,7 @@ requests = ">=2.0" name = "filelock" version = "3.12.2" description = "A platform independent file lock." -category = "main" -optional = true +optional = false python-versions = ">=3.7" files = [ {file = "filelock-3.12.2-py3-none-any.whl", hash = "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec"}, @@ -1505,7 +1427,6 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "diff-cover (>=7.5)", "p name = "fire" version = "0.5.0" description = "A library for automatically generating command line interfaces." -category = "main" optional = false python-versions = "*" files = [ @@ -1520,7 +1441,6 @@ termcolor = "*" name = "flatbuffers" version = "23.5.26" description = "The FlatBuffers serialization format for Python" -category = "main" optional = false python-versions = "*" files = [ @@ -1532,7 +1452,6 @@ files = [ name = "fonttools" version = "4.40.0" description = "Tools to manipulate font files" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1590,7 +1509,6 @@ woff = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "zopfli (>=0.1.4)"] name = "freezegun" version = "1.2.2" description = "Let your Python tests travel through time" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1605,7 +1523,6 @@ python-dateutil = ">=2.7" name = "frozenlist" version = "1.3.3" description = "A list-like structure which implements collections.abc.MutableSequence" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1689,7 +1606,6 @@ files = [ name = "fsspec" version = "2023.6.0" description = "File-system specification" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1725,7 +1641,6 @@ tqdm = ["tqdm"] name = "future" version = "0.18.3" description = "Clean single-source support for Python 3 and 2" -category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1736,7 +1651,6 @@ files = [ name = "gast" version = "0.4.0" description = "Python AST that abstracts the underlying Python version" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1748,7 +1662,6 @@ files = [ name = "gitdb" version = "4.0.10" description = "Git Object Database" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1763,7 +1676,6 @@ smmap = ">=3.0.1,<6" name = "github3-py" version = "3.2.0" description = "Python wrapper for the GitHub API(http://developer.github.com/v3)" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -1781,7 +1693,6 @@ uritemplate = ">=3.0.0" name = "gitpython" version = "3.1.31" description = "GitPython is a Python library used to interact with Git repositories" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1796,7 +1707,6 @@ gitdb = ">=4.0.1,<5" name = "google-api-core" version = "2.11.1" description = "Google API client core library" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1819,7 +1729,6 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] name = "google-auth" version = "2.21.0" description = "Google Authentication Library" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1845,7 +1754,6 @@ requests = ["requests (>=2.20.0,<3.0.0.dev0)"] name = "google-auth-oauthlib" version = "1.0.0" description = "Google Authentication Library" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1864,7 +1772,6 @@ tool = ["click (>=6.0.0)"] name = "google-cloud-core" version = "2.3.3" description = "Google Cloud API client core library" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1873,7 +1780,7 @@ files = [ ] [package.dependencies] -google-api-core = ">=1.31.6,<2.0.0 || >2.3.0,<3.0.0dev" +google-api-core = ">=1.31.6,<2.0.dev0 || >2.3.0,<3.0.0dev" google-auth = ">=1.25.0,<3.0dev" [package.extras] @@ -1883,7 +1790,6 @@ grpc = ["grpcio (>=1.38.0,<2.0dev)"] name = "google-cloud-storage" version = "2.10.0" description = "Google Cloud Storage API client library" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1892,7 +1798,7 @@ files = [ ] [package.dependencies] -google-api-core = ">=1.31.5,<2.0.0 || >2.3.0,<3.0.0dev" +google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" google-auth = ">=1.25.0,<3.0dev" google-cloud-core = ">=2.3.0,<3.0dev" google-resumable-media = ">=2.3.2" @@ -1905,7 +1811,6 @@ protobuf = ["protobuf (<5.0.0dev)"] name = "google-crc32c" version = "1.5.0" description = "A python wrapper of the C library 'Google CRC32C'" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1986,7 +1891,6 @@ testing = ["pytest"] name = "google-pasta" version = "0.2.0" description = "pasta is an AST-based Python refactoring library" -category = "main" optional = false python-versions = "*" files = [ @@ -2002,7 +1906,6 @@ six = "*" name = "google-resumable-media" version = "2.5.0" description = "Utilities for Google Media Downloads and Resumable Uploads" -category = "dev" optional = false python-versions = ">= 3.7" files = [ @@ -2021,7 +1924,6 @@ requests = ["requests (>=2.18.0,<3.0.0dev)"] name = "googleapis-common-protos" version = "1.59.1" description = "Common protobufs used in Google APIs" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2039,7 +1941,6 @@ grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] name = "greenlet" version = "2.0.2" description = "Lightweight in-process concurrent programming" -category = "main" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" files = [ @@ -2117,7 +2018,6 @@ test = ["objgraph", "psutil"] name = "grpcio" version = "1.56.0" description = "HTTP/2-based RPC framework" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2175,7 +2075,6 @@ protobuf = ["grpcio-tools (>=1.56.0)"] name = "h11" version = "0.14.0" description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2187,7 +2086,6 @@ files = [ name = "h5py" version = "3.9.0" description = "Read and write HDF5 files from Python" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2221,7 +2119,6 @@ numpy = ">=1.17.3" name = "httpcore" version = "0.16.3" description = "A minimal low-level HTTP client." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2233,17 +2130,16 @@ files = [ anyio = ">=3.0,<5.0" certifi = "*" h11 = ">=0.13,<0.15" -sniffio = ">=1.0.0,<2.0.0" +sniffio = "==1.*" [package.extras] http2 = ["h2 (>=3,<5)"] -socks = ["socksio (>=1.0.0,<2.0.0)"] +socks = ["socksio (==1.*)"] [[package]] name = "httptools" version = "0.5.0" description = "A collection of framework independent HTTP protocol utils." -category = "main" optional = false python-versions = ">=3.5.0" files = [ @@ -2297,7 +2193,6 @@ test = ["Cython (>=0.29.24,<0.30.0)"] name = "httpx" version = "0.23.3" description = "The next generation HTTP client." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2313,25 +2208,24 @@ sniffio = "*" [package.extras] brotli = ["brotli", "brotlicffi"] -cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<13)"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<13)"] http2 = ["h2 (>=3,<5)"] -socks = ["socksio (>=1.0.0,<2.0.0)"] +socks = ["socksio (==1.*)"] [[package]] name = "huggingface-hub" -version = "0.16.2" +version = "0.27.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" -category = "main" -optional = true -python-versions = ">=3.7.0" +optional = false +python-versions = ">=3.8.0" files = [ - {file = "huggingface_hub-0.16.2-py3-none-any.whl", hash = "sha256:92facff575c11a8cf4b35d184ae67867a577a1b30865edcd8a9c5a48d2202133"}, - {file = "huggingface_hub-0.16.2.tar.gz", hash = "sha256:205abbf02a3408129a309f09e6d1a88d0c82de296b498682a813d9baa91c272f"}, + {file = "huggingface_hub-0.27.0-py3-none-any.whl", hash = "sha256:8f2e834517f1f1ddf1ecc716f91b120d7333011b7485f665a9a412eacb1a2a81"}, + {file = "huggingface_hub-0.27.0.tar.gz", hash = "sha256:902cce1a1be5739f5589e560198a65a8edcfd3b830b1666f36e4b961f0454fac"}, ] [package.dependencies] filelock = "*" -fsspec = "*" +fsspec = ">=2023.5.0" packaging = ">=20.9" pyyaml = ">=5.1" requests = "*" @@ -2339,22 +2233,23 @@ tqdm = ">=4.42.1" typing-extensions = ">=3.7.4.3" [package.extras] -all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "numpy", "pydantic", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "urllib3 (<2.0)"] +all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "libcst (==1.4.0)", "mypy (==1.5.1)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.5.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] cli = ["InquirerPy (==0.3.4)"] -dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "numpy", "pydantic", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "urllib3 (<2.0)"] +dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "libcst (==1.4.0)", "mypy (==1.5.1)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.5.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] -inference = ["aiohttp", "pydantic"] -quality = ["black (>=23.1,<24.0)", "mypy (==0.982)", "ruff (>=0.0.241)"] +hf-transfer = ["hf-transfer (>=0.1.4)"] +inference = ["aiohttp"] +quality = ["libcst (==1.4.0)", "mypy (==1.5.1)", "ruff (>=0.5.0)"] tensorflow = ["graphviz", "pydot", "tensorflow"] -testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "gradio", "jedi", "numpy", "pydantic", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] -torch = ["torch"] -typing = ["pydantic", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"] +tensorflow-testing = ["keras (<3.0)", "tensorflow"] +testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] +torch = ["safetensors[torch]", "torch"] +typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] [[package]] name = "humanfriendly" version = "10.0" description = "Human friendly output for text interfaces using Python" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -2369,7 +2264,6 @@ pyreadline3 = {version = "*", markers = "sys_platform == \"win32\" and python_ve name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -2381,7 +2275,6 @@ files = [ name = "importlib-metadata" version = "6.7.0" description = "Read metadata from Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2401,7 +2294,6 @@ testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs name = "importlib-resources" version = "5.12.0" description = "Read resources from Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2420,7 +2312,6 @@ testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-chec name = "incremental" version = "22.10.0" description = "\"A small library that versions your Python projects.\"" -category = "dev" optional = false python-versions = "*" files = [ @@ -2436,7 +2327,6 @@ scripts = ["click (>=6.0)", "twisted (>=16.4.0)"] name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2448,7 +2338,6 @@ files = [ name = "isodate" version = "0.6.1" description = "An ISO 8601 date/time/duration parser and formatter" -category = "dev" optional = false python-versions = "*" files = [ @@ -2463,7 +2352,6 @@ six = "*" name = "jax" version = "0.4.13" description = "Differentiate, compile, and transform Numpy code." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2494,7 +2382,6 @@ tpu = ["jaxlib (==0.4.13)", "libtpu-nightly (==0.1.dev20230622)"] name = "jieba" version = "0.42.1" description = "Chinese Words Segmentation Utilities" -category = "main" optional = true python-versions = "*" files = [ @@ -2505,7 +2392,6 @@ files = [ name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2523,7 +2409,6 @@ i18n = ["Babel (>=2.7)"] name = "jmespath" version = "1.0.1" description = "JSON Matching Expressions" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2533,38 +2418,35 @@ files = [ [[package]] name = "joblib" -version = "1.2.0" +version = "1.4.2" description = "Lightweight pipelining with Python functions" -category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "joblib-1.2.0-py3-none-any.whl", hash = "sha256:091138ed78f800342968c523bdde947e7a305b8594b910a0fea2ab83c3c6d385"}, - {file = "joblib-1.2.0.tar.gz", hash = "sha256:e1cee4a79e4af22881164f218d4311f60074197fb707e082e803b61f6d137018"}, + {file = "joblib-1.4.2-py3-none-any.whl", hash = "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6"}, + {file = "joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"}, ] [[package]] name = "jsonpickle" -version = "3.0.1" -description = "Python library for serializing any arbitrary object graph into JSON" -category = "main" +version = "3.0.4" +description = "Serialize any Python object to JSON" optional = false python-versions = ">=3.7" files = [ - {file = "jsonpickle-3.0.1-py2.py3-none-any.whl", hash = "sha256:130d8b293ea0add3845de311aaba55e6d706d0bb17bc123bd2c8baf8a39ac77c"}, - {file = "jsonpickle-3.0.1.tar.gz", hash = "sha256:032538804795e73b94ead410800ac387fdb6de98f8882ac957fcd247e3a85200"}, + {file = "jsonpickle-3.0.4-py3-none-any.whl", hash = "sha256:04ae7567a14269579e3af66b76bda284587458d7e8a204951ca8f71a3309952e"}, + {file = "jsonpickle-3.0.4.tar.gz", hash = "sha256:a1b14c8d6221cd8f394f2a97e735ea1d7edc927fbd135b26f2f8700657c8c62b"}, ] [package.extras] -docs = ["jaraco.packaging (>=3.2)", "rst.linker (>=1.9)", "sphinx"] -testing = ["ecdsa", "feedparser", "gmpy2", "numpy", "pandas", "pymongo", "pytest (>=3.5,!=3.7.3)", "pytest-black-multipy", "pytest-checkdocs (>=1.2.3)", "pytest-cov", "pytest-flake8 (>=1.1.1)", "scikit-learn", "sqlalchemy"] -testing-libs = ["simplejson", "ujson"] +docs = ["furo", "rst.linker (>=1.9)", "sphinx"] +packaging = ["build", "twine"] +testing = ["bson", "ecdsa", "feedparser", "gmpy2", "numpy", "pandas", "pymongo", "pytest (>=3.5,!=3.7.3)", "pytest-benchmark", "pytest-benchmark[histogram]", "pytest-checkdocs (>=1.2.3)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-ruff (>=0.2.1)", "scikit-learn", "scipy", "scipy (>=1.9.3)", "simplejson", "sqlalchemy", "ujson"] [[package]] name = "jsonschema" version = "4.17.3" description = "An implementation of JSON Schema validation for Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2586,7 +2468,6 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- name = "keras" version = "2.12.0" description = "Deep learning for humans." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2597,7 +2478,6 @@ files = [ name = "kiwisolver" version = "1.4.4" description = "A fast implementation of the Cassowary constraint solver" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2675,7 +2555,6 @@ files = [ name = "langcodes" version = "3.3.0" description = "Tools for labeling human languages with IETF language tags" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -2690,7 +2569,6 @@ data = ["language-data (>=1.1,<2.0)"] name = "libclang" version = "16.0.0" description = "Clang Python Bindings, mirrored from the official LLVM repo: https://github.com/llvm/llvm-project/tree/main/clang/bindings/python, to make the installation process easier." -category = "main" optional = false python-versions = "*" files = [ @@ -2708,7 +2586,6 @@ files = [ name = "locket" version = "1.0.0" description = "File-based locks for Python on Linux and Windows" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -2716,23 +2593,10 @@ files = [ {file = "locket-1.0.0.tar.gz", hash = "sha256:5c0d4c052a8bbbf750e056a8e65ccd309086f4f0f18a2eac306a8dfa4112a632"}, ] -[[package]] -name = "magic-filter" -version = "1.0.9" -description = "This package provides magic filter based on dynamic attribute getter" -category = "main" -optional = false -python-versions = ">=3.7,<4.0" -files = [ - {file = "magic-filter-1.0.9.tar.gz", hash = "sha256:d0f1ffa5ff1fbe5105fd5f293c79b5d3795f336ea0f6129c636959a687bf422a"}, - {file = "magic_filter-1.0.9-py3-none-any.whl", hash = "sha256:51002312a8972fa514b998b7ff89340c98be3fc499967c1f5f2af98d13baf8d5"}, -] - [[package]] name = "markdown" version = "3.4.3" description = "Python implementation of John Gruber's Markdown." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2750,7 +2614,6 @@ testing = ["coverage", "pyyaml"] name = "markdown-it-py" version = "3.0.0" description = "Python port of markdown-it. Markdown parsing, done right!" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2775,7 +2638,6 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] name = "markupsafe" version = "2.1.3" description = "Safely add untrusted strings to HTML/XML markup." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2845,7 +2707,6 @@ files = [ name = "matplotlib" version = "3.5.3" description = "Python plotting package" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2900,7 +2761,6 @@ python-dateutil = ">=2.7" name = "mattermostwrapper" version = "2.2" description = "A mattermost api v4 wrapper to interact with api" -category = "main" optional = false python-versions = "*" files = [ @@ -2914,7 +2774,6 @@ requests = "*" name = "mdurl" version = "0.1.2" description = "Markdown URL utilities" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2926,7 +2785,6 @@ files = [ name = "memory-profiler" version = "0.61.0" description = "A module for monitoring memory usage of a python program" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -2941,7 +2799,6 @@ psutil = "*" name = "ml-dtypes" version = "0.2.0" description = "" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2966,8 +2823,8 @@ files = [ [package.dependencies] numpy = [ - {version = ">1.20", markers = "python_version <= \"3.9\""}, {version = ">=1.21.2", markers = "python_version > \"3.9\""}, + {version = ">1.20", markers = "python_version <= \"3.9\""}, ] [package.extras] @@ -2977,7 +2834,6 @@ dev = ["absl-py", "pyink", "pylint (>=2.6.0)", "pytest", "pytest-xdist"] name = "mongomock" version = "4.1.2" description = "Fake pymongo stub for testing simple MongoDB-dependent code" -category = "dev" optional = false python-versions = "*" files = [ @@ -2993,7 +2849,6 @@ sentinels = "*" name = "monotonic" version = "1.6" description = "An implementation of time.monotonic() for Python 2 & < 3.3" -category = "dev" optional = false python-versions = "*" files = [ @@ -3005,7 +2860,6 @@ files = [ name = "moto" version = "4.1.12" description = "" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3052,7 +2906,6 @@ xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] name = "msgpack" version = "1.0.5" description = "MessagePack serializer" -category = "main" optional = false python-versions = "*" files = [ @@ -3125,7 +2978,6 @@ files = [ name = "multidict" version = "5.2.0" description = "multidict implementation" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -3207,7 +3059,6 @@ files = [ name = "murmurhash" version = "1.0.9" description = "Cython bindings for MurmurHash" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -3245,7 +3096,6 @@ files = [ name = "mypy" version = "1.0.1" description = "Optional static typing for Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3292,7 +3142,6 @@ reports = ["lxml"] name = "mypy-extensions" version = "0.4.4" description = "Experimental type system extensions for programs checked with the mypy typechecker." -category = "dev" optional = false python-versions = ">=2.7" files = [ @@ -3303,7 +3152,6 @@ files = [ name = "networkx" version = "2.6.3" description = "Python package for creating and manipulating graphs and networks" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3322,7 +3170,6 @@ test = ["codecov (>=2.1)", "pytest (>=6.2)", "pytest-cov (>=2.12)"] name = "nr-util" version = "0.8.12" description = "General purpose Python utility library." -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -3338,7 +3185,6 @@ typing-extensions = ">=3.0.0" name = "numpy" version = "1.22.3" description = "NumPy is the fundamental package for array computing with Python." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -3368,7 +3214,6 @@ files = [ name = "numpy" version = "1.23.5" description = "NumPy is the fundamental package for array computing with Python." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -3406,7 +3251,6 @@ files = [ name = "oauthlib" version = "3.2.2" description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -3423,7 +3267,6 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] name = "opt-einsum" version = "3.3.0" description = "Optimizing numpys einsum function" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -3442,7 +3285,6 @@ tests = ["pytest", "pytest-cov", "pytest-pep8"] name = "packaging" version = "20.9" description = "Core utilities for Python packages" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -3457,7 +3299,6 @@ pyparsing = ">=2.0.2" name = "pamqp" version = "3.2.1" description = "RabbitMQ Focused AMQP low-level library" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3473,7 +3314,6 @@ testing = ["coverage", "flake8", "flake8-comprehensions", "flake8-deprecated", " name = "partd" version = "1.4.0" description = "Appendable key-value storage" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3492,7 +3332,6 @@ complete = ["blosc", "numpy (>=1.9.0)", "pandas (>=0.19.0)", "pyzmq"] name = "pathspec" version = "0.11.1" description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3504,7 +3343,6 @@ files = [ name = "pathy" version = "0.10.2" description = "pathlib.Path subclasses for local and cloud bucket storage" -category = "main" optional = true python-versions = ">= 3.6" files = [ @@ -3527,7 +3365,6 @@ test = ["mock", "pytest", "pytest-coverage", "typer-cli"] name = "pbr" version = "5.11.1" description = "Python Build Reasonableness" -category = "dev" optional = false python-versions = ">=2.6" files = [ @@ -3539,7 +3376,6 @@ files = [ name = "pep440-version-utils" version = "0.3.0" description = "Utilities to deal with pep440 versioning" -category = "dev" optional = false python-versions = ">=3.6,<4.0" files = [ @@ -3554,7 +3390,6 @@ packaging = ">=20.3,<21.0" name = "pillow" version = "10.0.1" description = "Python Imaging Library (Fork)" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -3622,7 +3457,6 @@ tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "pa name = "pkgutil-resolve-name" version = "1.3.10" description = "Resolve a name to an object." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -3634,7 +3468,6 @@ files = [ name = "platformdirs" version = "3.8.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3650,7 +3483,6 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest- name = "pluggy" version = "1.2.0" description = "plugin and hook calling mechanisms for python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3666,7 +3498,6 @@ testing = ["pytest", "pytest-benchmark"] name = "portalocker" version = "2.7.0" description = "Wraps the portalocker recipe for easy usage" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -3686,7 +3517,6 @@ tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "p name = "preshed" version = "3.0.8" description = "Cython hash table that trusts the keys are pre-hashed" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -3728,7 +3558,6 @@ murmurhash = ">=0.28.0,<1.1.0" name = "prompt-toolkit" version = "3.0.28" description = "Library for building powerful interactive command lines in Python" -category = "main" optional = false python-versions = ">=3.6.2" files = [ @@ -3743,7 +3572,6 @@ wcwidth = "*" name = "protobuf" version = "4.23.3" description = "" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3766,7 +3594,6 @@ files = [ name = "psutil" version = "5.9.5" description = "Cross-platform lib for process and system monitoring in Python." -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -3793,7 +3620,6 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] name = "psycopg2-binary" version = "2.9.6" description = "psycopg2 - Python-PostgreSQL Database Adapter" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -3865,7 +3691,6 @@ files = [ name = "pyasn1" version = "0.5.0" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -3877,7 +3702,6 @@ files = [ name = "pyasn1-modules" version = "0.3.0" description = "A collection of ASN.1-based protocols modules" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -3892,7 +3716,6 @@ pyasn1 = ">=0.4.6,<0.6.0" name = "pycparser" version = "2.21" description = "C parser in Python" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -3904,7 +3727,6 @@ files = [ name = "pydantic" version = "1.10.9" description = "Data validation and settings management using python type hints" -category = "main" optional = true python-versions = ">=3.7" files = [ @@ -3957,7 +3779,6 @@ email = ["email-validator (>=1.0.3)"] name = "pydoc-markdown" version = "4.7.0" description = "Create Python API documentation in Markdown format." -category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -3984,7 +3805,6 @@ yapf = ">=0.30.0" name = "pydot" version = "1.4.2" description = "Python interface to Graphviz's Dot" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -3999,7 +3819,6 @@ pyparsing = ">=2.1.4" name = "pygments" version = "2.15.1" description = "Pygments is a syntax highlighting package written in Python." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4014,7 +3833,6 @@ plugins = ["importlib-metadata"] name = "pyjwt" version = "2.7.0" description = "JSON Web Token implementation in Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -4035,7 +3853,6 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] name = "pykwalify" version = "1.8.0" description = "Python lib/cli for JSON/YAML schema validation" -category = "main" optional = false python-versions = "*" files = [ @@ -4052,7 +3869,6 @@ python-dateutil = ">=2.8.0" name = "pymongo" version = "4.3.3" description = "Python driver for MongoDB " -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -4147,7 +3963,6 @@ zstd = ["zstandard"] name = "pyparsing" version = "3.1.0" description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "main" optional = false python-versions = ">=3.6.8" files = [ @@ -4162,7 +3977,6 @@ diagrams = ["jinja2", "railroad-diagrams"] name = "pyreadline3" version = "3.4.1" description = "A python implementation of GNU readline." -category = "main" optional = false python-versions = "*" files = [ @@ -4174,7 +3988,6 @@ files = [ name = "pyrsistent" version = "0.19.3" description = "Persistent/Functional/Immutable data structures" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -4211,7 +4024,6 @@ files = [ name = "pytest" version = "7.4.0" description = "pytest: simple powerful testing with Python" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4234,7 +4046,6 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no name = "pytest-asyncio" version = "0.20.3" description = "Pytest support for asyncio" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4253,7 +4064,6 @@ testing = ["coverage (>=6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy name = "pytest-cov" version = "4.1.0" description = "Pytest plugin for measuring coverage." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4272,7 +4082,6 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale name = "pytest-sanic" version = "1.9.1" description = "a pytest plugin for Sanic" -category = "dev" optional = false python-versions = ">=3.7" files = [] @@ -4294,7 +4103,6 @@ resolved_reference = "4092e8005fbbdc29b892cd8b09399894d86b1ac7" name = "pytest-timeout" version = "2.1.0" description = "pytest plugin to abort hanging tests" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -4309,7 +4117,6 @@ pytest = ">=5.0.0" name = "pytest-xdist" version = "3.3.1" description = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4330,7 +4137,6 @@ testing = ["filelock"] name = "python-crfsuite" version = "0.9.9" description = "Python binding for CRFsuite" -category = "main" optional = false python-versions = "*" files = [ @@ -4347,6 +4153,15 @@ files = [ {file = "python_crfsuite-0.9.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ab333b7cda21b1b5ab76f8e16e5f00654360df057e2092b273681d64850f714"}, {file = "python_crfsuite-0.9.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ee6153ed8a26adaea645445b997c55d67505476976dfc40f4bbd46200b66de15"}, {file = "python_crfsuite-0.9.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c32292f722e293cee87aac0d793abd8b53dc05126e4b6f0202c41e0e7d027005"}, + {file = "python_crfsuite-0.9.9-cp311-cp311-win32.whl", hash = "sha256:3e8bbacff1d86cbc18e1d52f617c85521029127a09c86ed428cd8238384a9db3"}, + {file = "python_crfsuite-0.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:83f87b652108110263aa83c220baccc36c911f04cf422cf6632a5f42121bce6a"}, + {file = "python_crfsuite-0.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:92f72eb554dac53218805958747b4cd417bd76039f083f66cc9881987d88e167"}, + {file = "python_crfsuite-0.9.9-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5cdd1e823fa9dfe611a573a0c1371941e887ad8b8ffbc25e2d87e4cd6d4f22af"}, + {file = "python_crfsuite-0.9.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54f94d6fabb14ad8106dc65b5d38665bb0abb16527d4a6aa2ba233670c3480db"}, + {file = "python_crfsuite-0.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:65ebfab892c49c49b5e1030318d144c559e449bdf86b12983fa7ba0e88f7abdf"}, + {file = "python_crfsuite-0.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5c1bb3d2802bb777affbe2855fc723aa32345ead2fe26ad18c7e1417bc104c53"}, + {file = "python_crfsuite-0.9.9-cp312-cp312-win32.whl", hash = "sha256:abef974a2b520c0204cb15b0b799fdbc1c1a0af4be2b9ad7548800de95975345"}, + {file = "python_crfsuite-0.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:ccc4c4f1cd47c74553d03d915c7dd7c06fc23b41310f30f35f2e5c09cdeb9297"}, {file = "python_crfsuite-0.9.9-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:01d279f1c8225aaf66290563312708c1905dce84f70ee5e374ecfb2dec1c2343"}, {file = "python_crfsuite-0.9.9-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5f29e94fcb9e2f8f52c7323668752b34dcffef91751b0d1e0789ecbbc0069842"}, {file = "python_crfsuite-0.9.9-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:497a8b9c0df152ada10732c07853942a8725cb66fe5b6fe1a64f768ecf583291"}, @@ -4381,7 +4196,6 @@ files = [ name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -4396,7 +4210,6 @@ six = ">=1.5" name = "python-engineio" version = "4.5.1" description = "Engine.IO server and client for Python" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -4413,7 +4226,6 @@ docs = ["sphinx"] name = "python-socketio" version = "5.8.0" description = "Socket.IO server and client for Python" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -4433,7 +4245,6 @@ client = ["requests (>=2.21.0)", "websocket-client (>=0.54.0)"] name = "pytz" version = "2022.7.1" description = "World timezone definitions, modern and historical" -category = "main" optional = false python-versions = "*" files = [ @@ -4445,7 +4256,6 @@ files = [ name = "pywin32" version = "306" description = "Python for Window Extensions" -category = "main" optional = false python-versions = "*" files = [ @@ -4469,7 +4279,6 @@ files = [ name = "pyyaml" version = "6.0.1" description = "YAML parser and emitter for Python" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -4491,6 +4300,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -4529,7 +4339,6 @@ files = [ name = "questionary" version = "1.10.0" description = "Python library to build pretty command line user prompts โญ๏ธ" -category = "main" optional = false python-versions = ">=3.6,<4.0" files = [ @@ -4547,7 +4356,6 @@ docs = ["Sphinx (>=3.3,<4.0)", "sphinx-autobuild (>=2020.9.1,<2021.0.0)", "sphin name = "randomname" version = "0.1.5" description = "Generate random adj-noun names like docker and github." -category = "main" optional = false python-versions = "*" files = [ @@ -4561,7 +4369,6 @@ fire = "*" name = "rasa-sdk" version = "3.6.2" description = "Open source machine learning framework to automate text- and voice-based conversations: NLU, dialogue management, connect to Slack, Facebook, and more - Create chatbots and voice assistants" -category = "main" optional = false python-versions = ">=3.8,<3.11" files = [ @@ -4585,7 +4392,6 @@ wheel = ">=0.38.1" name = "redis" version = "4.6.0" description = "Python client for Redis database and key-value store" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -4604,7 +4410,6 @@ ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)" name = "regex" version = "2022.10.31" description = "Alternative regular expression module, to replace re." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -4702,7 +4507,6 @@ files = [ name = "requests" version = "2.31.0" description = "Python HTTP for Humans." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -4724,7 +4528,6 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "requests-oauthlib" version = "1.3.1" description = "OAuthlib authentication support for Requests." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -4743,7 +4546,6 @@ rsa = ["oauthlib[signedtoken] (>=3.0.0)"] name = "requests-toolbelt" version = "1.0.0" description = "A utility belt for advanced users of python-requests" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -4758,7 +4560,6 @@ requests = ">=2.0.1,<3.0.0" name = "responses" version = "0.22.0" description = "A utility library for mocking out the `requests` Python library." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4779,7 +4580,6 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy name = "rfc3986" version = "1.5.0" description = "Validating URI References per RFC 3986" -category = "dev" optional = false python-versions = "*" files = [ @@ -4797,7 +4597,6 @@ idna2008 = ["idna"] name = "rich" version = "13.4.2" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -category = "dev" optional = false python-versions = ">=3.7.0" files = [ @@ -4817,7 +4616,6 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] name = "rocketchat-api" version = "1.30.0" description = "Python API wrapper for Rocket.Chat" -category = "main" optional = false python-versions = "*" files = [ @@ -4833,7 +4631,6 @@ requests = "*" name = "rsa" version = "4.9" description = "Pure-Python RSA implementation" -category = "main" optional = false python-versions = ">=3.6,<4" files = [ @@ -4848,7 +4645,6 @@ pyasn1 = ">=0.1.3" name = "ruamel-yaml" version = "0.17.21" description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" -category = "main" optional = false python-versions = ">=3" files = [ @@ -4867,7 +4663,6 @@ jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] name = "ruamel-yaml-clib" version = "0.2.7" description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -4914,7 +4709,6 @@ files = [ name = "ruff" version = "0.0.255" description = "An extremely fast Python linter, written in Rust." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4941,7 +4735,6 @@ files = [ name = "s3transfer" version = "0.6.1" description = "An Amazon S3 Transfer Manager" -category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -4955,11 +4748,142 @@ botocore = ">=1.12.36,<2.0a.0" [package.extras] crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] +[[package]] +name = "safetensors" +version = "0.4.5" +description = "" +optional = false +python-versions = ">=3.7" +files = [ + {file = "safetensors-0.4.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a63eaccd22243c67e4f2b1c3e258b257effc4acd78f3b9d397edc8cf8f1298a7"}, + {file = "safetensors-0.4.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:23fc9b4ec7b602915cbb4ec1a7c1ad96d2743c322f20ab709e2c35d1b66dad27"}, + {file = "safetensors-0.4.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6885016f34bef80ea1085b7e99b3c1f92cb1be78a49839203060f67b40aee761"}, + {file = "safetensors-0.4.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:133620f443450429322f238fda74d512c4008621227fccf2f8cf4a76206fea7c"}, + {file = "safetensors-0.4.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4fb3e0609ec12d2a77e882f07cced530b8262027f64b75d399f1504ffec0ba56"}, + {file = "safetensors-0.4.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d0f1dd769f064adc33831f5e97ad07babbd728427f98e3e1db6902e369122737"}, + {file = "safetensors-0.4.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6d156bdb26732feada84f9388a9f135528c1ef5b05fae153da365ad4319c4c5"}, + {file = "safetensors-0.4.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9e347d77e2c77eb7624400ccd09bed69d35c0332f417ce8c048d404a096c593b"}, + {file = "safetensors-0.4.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9f556eea3aec1d3d955403159fe2123ddd68e880f83954ee9b4a3f2e15e716b6"}, + {file = "safetensors-0.4.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9483f42be3b6bc8ff77dd67302de8ae411c4db39f7224dec66b0eb95822e4163"}, + {file = "safetensors-0.4.5-cp310-none-win32.whl", hash = "sha256:7389129c03fadd1ccc37fd1ebbc773f2b031483b04700923c3511d2a939252cc"}, + {file = "safetensors-0.4.5-cp310-none-win_amd64.whl", hash = "sha256:e98ef5524f8b6620c8cdef97220c0b6a5c1cef69852fcd2f174bb96c2bb316b1"}, + {file = "safetensors-0.4.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:21f848d7aebd5954f92538552d6d75f7c1b4500f51664078b5b49720d180e47c"}, + {file = "safetensors-0.4.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bb07000b19d41e35eecef9a454f31a8b4718a185293f0d0b1c4b61d6e4487971"}, + {file = "safetensors-0.4.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09dedf7c2fda934ee68143202acff6e9e8eb0ddeeb4cfc24182bef999efa9f42"}, + {file = "safetensors-0.4.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:59b77e4b7a708988d84f26de3ebead61ef1659c73dcbc9946c18f3b1786d2688"}, + {file = "safetensors-0.4.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5d3bc83e14d67adc2e9387e511097f254bd1b43c3020440e708858c684cbac68"}, + {file = "safetensors-0.4.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:39371fc551c1072976073ab258c3119395294cf49cdc1f8476794627de3130df"}, + {file = "safetensors-0.4.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6c19feda32b931cae0acd42748a670bdf56bee6476a046af20181ad3fee4090"}, + {file = "safetensors-0.4.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a659467495de201e2f282063808a41170448c78bada1e62707b07a27b05e6943"}, + {file = "safetensors-0.4.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bad5e4b2476949bcd638a89f71b6916fa9a5cae5c1ae7eede337aca2100435c0"}, + {file = "safetensors-0.4.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a3a315a6d0054bc6889a17f5668a73f94f7fe55121ff59e0a199e3519c08565f"}, + {file = "safetensors-0.4.5-cp311-none-win32.whl", hash = "sha256:a01e232e6d3d5cf8b1667bc3b657a77bdab73f0743c26c1d3c5dd7ce86bd3a92"}, + {file = "safetensors-0.4.5-cp311-none-win_amd64.whl", hash = "sha256:cbd39cae1ad3e3ef6f63a6f07296b080c951f24cec60188378e43d3713000c04"}, + {file = "safetensors-0.4.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:473300314e026bd1043cef391bb16a8689453363381561b8a3e443870937cc1e"}, + {file = "safetensors-0.4.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:801183a0f76dc647f51a2d9141ad341f9665602a7899a693207a82fb102cc53e"}, + {file = "safetensors-0.4.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1524b54246e422ad6fb6aea1ac71edeeb77666efa67230e1faf6999df9b2e27f"}, + {file = "safetensors-0.4.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b3139098e3e8b2ad7afbca96d30ad29157b50c90861084e69fcb80dec7430461"}, + {file = "safetensors-0.4.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65573dc35be9059770808e276b017256fa30058802c29e1038eb1c00028502ea"}, + {file = "safetensors-0.4.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fd33da8e9407559f8779c82a0448e2133737f922d71f884da27184549416bfed"}, + {file = "safetensors-0.4.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3685ce7ed036f916316b567152482b7e959dc754fcc4a8342333d222e05f407c"}, + {file = "safetensors-0.4.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dde2bf390d25f67908278d6f5d59e46211ef98e44108727084d4637ee70ab4f1"}, + {file = "safetensors-0.4.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7469d70d3de970b1698d47c11ebbf296a308702cbaae7fcb993944751cf985f4"}, + {file = "safetensors-0.4.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3a6ba28118636a130ccbb968bc33d4684c48678695dba2590169d5ab03a45646"}, + {file = "safetensors-0.4.5-cp312-none-win32.whl", hash = "sha256:c859c7ed90b0047f58ee27751c8e56951452ed36a67afee1b0a87847d065eec6"}, + {file = "safetensors-0.4.5-cp312-none-win_amd64.whl", hash = "sha256:b5a8810ad6a6f933fff6c276eae92c1da217b39b4d8b1bc1c0b8af2d270dc532"}, + {file = "safetensors-0.4.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:25e5f8e2e92a74f05b4ca55686234c32aac19927903792b30ee6d7bd5653d54e"}, + {file = "safetensors-0.4.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:81efb124b58af39fcd684254c645e35692fea81c51627259cdf6d67ff4458916"}, + {file = "safetensors-0.4.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:585f1703a518b437f5103aa9cf70e9bd437cb78eea9c51024329e4fb8a3e3679"}, + {file = "safetensors-0.4.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4b99fbf72e3faf0b2f5f16e5e3458b93b7d0a83984fe8d5364c60aa169f2da89"}, + {file = "safetensors-0.4.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b17b299ca9966ca983ecda1c0791a3f07f9ca6ab5ded8ef3d283fff45f6bcd5f"}, + {file = "safetensors-0.4.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:76ded72f69209c9780fdb23ea89e56d35c54ae6abcdec67ccb22af8e696e449a"}, + {file = "safetensors-0.4.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2783956926303dcfeb1de91a4d1204cd4089ab441e622e7caee0642281109db3"}, + {file = "safetensors-0.4.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d94581aab8c6b204def4d7320f07534d6ee34cd4855688004a4354e63b639a35"}, + {file = "safetensors-0.4.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:67e1e7cb8678bb1b37ac48ec0df04faf689e2f4e9e81e566b5c63d9f23748523"}, + {file = "safetensors-0.4.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:dbd280b07e6054ea68b0cb4b16ad9703e7d63cd6890f577cb98acc5354780142"}, + {file = "safetensors-0.4.5-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:77d9b228da8374c7262046a36c1f656ba32a93df6cc51cd4453af932011e77f1"}, + {file = "safetensors-0.4.5-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:500cac01d50b301ab7bb192353317035011c5ceeef0fca652f9f43c000bb7f8d"}, + {file = "safetensors-0.4.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:75331c0c746f03158ded32465b7d0b0e24c5a22121743662a2393439c43a45cf"}, + {file = "safetensors-0.4.5-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:670e95fe34e0d591d0529e5e59fd9d3d72bc77b1444fcaa14dccda4f36b5a38b"}, + {file = "safetensors-0.4.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:098923e2574ff237c517d6e840acada8e5b311cb1fa226019105ed82e9c3b62f"}, + {file = "safetensors-0.4.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13ca0902d2648775089fa6a0c8fc9e6390c5f8ee576517d33f9261656f851e3f"}, + {file = "safetensors-0.4.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f0032bedc869c56f8d26259fe39cd21c5199cd57f2228d817a0e23e8370af25"}, + {file = "safetensors-0.4.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f4b15f51b4f8f2a512341d9ce3475cacc19c5fdfc5db1f0e19449e75f95c7dc8"}, + {file = "safetensors-0.4.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f6594d130d0ad933d885c6a7b75c5183cb0e8450f799b80a39eae2b8508955eb"}, + {file = "safetensors-0.4.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:60c828a27e852ded2c85fc0f87bf1ec20e464c5cd4d56ff0e0711855cc2e17f8"}, + {file = "safetensors-0.4.5-cp37-none-win32.whl", hash = "sha256:6d3de65718b86c3eeaa8b73a9c3d123f9307a96bbd7be9698e21e76a56443af5"}, + {file = "safetensors-0.4.5-cp37-none-win_amd64.whl", hash = "sha256:5a2d68a523a4cefd791156a4174189a4114cf0bf9c50ceb89f261600f3b2b81a"}, + {file = "safetensors-0.4.5-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:e7a97058f96340850da0601a3309f3d29d6191b0702b2da201e54c6e3e44ccf0"}, + {file = "safetensors-0.4.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:63bfd425e25f5c733f572e2246e08a1c38bd6f2e027d3f7c87e2e43f228d1345"}, + {file = "safetensors-0.4.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3664ac565d0e809b0b929dae7ccd74e4d3273cd0c6d1220c6430035befb678e"}, + {file = "safetensors-0.4.5-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:313514b0b9b73ff4ddfb4edd71860696dbe3c1c9dc4d5cc13dbd74da283d2cbf"}, + {file = "safetensors-0.4.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31fa33ee326f750a2f2134a6174773c281d9a266ccd000bd4686d8021f1f3dac"}, + {file = "safetensors-0.4.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:09566792588d77b68abe53754c9f1308fadd35c9f87be939e22c623eaacbed6b"}, + {file = "safetensors-0.4.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309aaec9b66cbf07ad3a2e5cb8a03205663324fea024ba391594423d0f00d9fe"}, + {file = "safetensors-0.4.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:53946c5813b8f9e26103c5efff4a931cc45d874f45229edd68557ffb35ffb9f8"}, + {file = "safetensors-0.4.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:868f9df9e99ad1e7f38c52194063a982bc88fedc7d05096f4f8160403aaf4bd6"}, + {file = "safetensors-0.4.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:9cc9449bd0b0bc538bd5e268221f0c5590bc5c14c1934a6ae359d44410dc68c4"}, + {file = "safetensors-0.4.5-cp38-none-win32.whl", hash = "sha256:83c4f13a9e687335c3928f615cd63a37e3f8ef072a3f2a0599fa09f863fb06a2"}, + {file = "safetensors-0.4.5-cp38-none-win_amd64.whl", hash = "sha256:b98d40a2ffa560653f6274e15b27b3544e8e3713a44627ce268f419f35c49478"}, + {file = "safetensors-0.4.5-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:cf727bb1281d66699bef5683b04d98c894a2803442c490a8d45cd365abfbdeb2"}, + {file = "safetensors-0.4.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:96f1d038c827cdc552d97e71f522e1049fef0542be575421f7684756a748e457"}, + {file = "safetensors-0.4.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:139fbee92570ecea774e6344fee908907db79646d00b12c535f66bc78bd5ea2c"}, + {file = "safetensors-0.4.5-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c36302c1c69eebb383775a89645a32b9d266878fab619819ce660309d6176c9b"}, + {file = "safetensors-0.4.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d641f5b8149ea98deb5ffcf604d764aad1de38a8285f86771ce1abf8e74c4891"}, + {file = "safetensors-0.4.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b4db6a61d968de73722b858038c616a1bebd4a86abe2688e46ca0cc2d17558f2"}, + {file = "safetensors-0.4.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b75a616e02f21b6f1d5785b20cecbab5e2bd3f6358a90e8925b813d557666ec1"}, + {file = "safetensors-0.4.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:788ee7d04cc0e0e7f944c52ff05f52a4415b312f5efd2ee66389fb7685ee030c"}, + {file = "safetensors-0.4.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:87bc42bd04fd9ca31396d3ca0433db0be1411b6b53ac5a32b7845a85d01ffc2e"}, + {file = "safetensors-0.4.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4037676c86365a721a8c9510323a51861d703b399b78a6b4486a54a65a975fca"}, + {file = "safetensors-0.4.5-cp39-none-win32.whl", hash = "sha256:1500418454529d0ed5c1564bda376c4ddff43f30fce9517d9bee7bcce5a8ef50"}, + {file = "safetensors-0.4.5-cp39-none-win_amd64.whl", hash = "sha256:9d1a94b9d793ed8fe35ab6d5cea28d540a46559bafc6aae98f30ee0867000cab"}, + {file = "safetensors-0.4.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:fdadf66b5a22ceb645d5435a0be7a0292ce59648ca1d46b352f13cff3ea80410"}, + {file = "safetensors-0.4.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d42ffd4c2259f31832cb17ff866c111684c87bd930892a1ba53fed28370c918c"}, + {file = "safetensors-0.4.5-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd8a1f6d2063a92cd04145c7fd9e31a1c7d85fbec20113a14b487563fdbc0597"}, + {file = "safetensors-0.4.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:951d2fcf1817f4fb0ef0b48f6696688a4e852a95922a042b3f96aaa67eedc920"}, + {file = "safetensors-0.4.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6ac85d9a8c1af0e3132371d9f2d134695a06a96993c2e2f0bbe25debb9e3f67a"}, + {file = "safetensors-0.4.5-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:e3cec4a29eb7fe8da0b1c7988bc3828183080439dd559f720414450de076fcab"}, + {file = "safetensors-0.4.5-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:21742b391b859e67b26c0b2ac37f52c9c0944a879a25ad2f9f9f3cd61e7fda8f"}, + {file = "safetensors-0.4.5-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c7db3006a4915151ce1913652e907cdede299b974641a83fbc092102ac41b644"}, + {file = "safetensors-0.4.5-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f68bf99ea970960a237f416ea394e266e0361895753df06e3e06e6ea7907d98b"}, + {file = "safetensors-0.4.5-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8158938cf3324172df024da511839d373c40fbfaa83e9abf467174b2910d7b4c"}, + {file = "safetensors-0.4.5-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:540ce6c4bf6b58cb0fd93fa5f143bc0ee341c93bb4f9287ccd92cf898cc1b0dd"}, + {file = "safetensors-0.4.5-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:bfeaa1a699c6b9ed514bd15e6a91e74738b71125a9292159e3d6b7f0a53d2cde"}, + {file = "safetensors-0.4.5-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:01c8f00da537af711979e1b42a69a8ec9e1d7112f208e0e9b8a35d2c381085ef"}, + {file = "safetensors-0.4.5-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a0dd565f83b30f2ca79b5d35748d0d99dd4b3454f80e03dfb41f0038e3bdf180"}, + {file = "safetensors-0.4.5-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:023b6e5facda76989f4cba95a861b7e656b87e225f61811065d5c501f78cdb3f"}, + {file = "safetensors-0.4.5-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9633b663393d5796f0b60249549371e392b75a0b955c07e9c6f8708a87fc841f"}, + {file = "safetensors-0.4.5-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78dd8adfb48716233c45f676d6e48534d34b4bceb50162c13d1f0bdf6f78590a"}, + {file = "safetensors-0.4.5-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e8deb16c4321d61ae72533b8451ec4a9af8656d1c61ff81aa49f966406e4b68"}, + {file = "safetensors-0.4.5-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:52452fa5999dc50c4decaf0c53aa28371f7f1e0fe5c2dd9129059fbe1e1599c7"}, + {file = "safetensors-0.4.5-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d5f23198821e227cfc52d50fa989813513db381255c6d100927b012f0cfec63d"}, + {file = "safetensors-0.4.5-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f4beb84b6073b1247a773141a6331117e35d07134b3bb0383003f39971d414bb"}, + {file = "safetensors-0.4.5-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:68814d599d25ed2fdd045ed54d370d1d03cf35e02dce56de44c651f828fb9b7b"}, + {file = "safetensors-0.4.5-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0b6453c54c57c1781292c46593f8a37254b8b99004c68d6c3ce229688931a22"}, + {file = "safetensors-0.4.5-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:adaa9c6dead67e2dd90d634f89131e43162012479d86e25618e821a03d1eb1dc"}, + {file = "safetensors-0.4.5-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:73e7d408e9012cd17511b382b43547850969c7979efc2bc353f317abaf23c84c"}, + {file = "safetensors-0.4.5-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:775409ce0fcc58b10773fdb4221ed1eb007de10fe7adbdf8f5e8a56096b6f0bc"}, + {file = "safetensors-0.4.5-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:834001bed193e4440c4a3950a31059523ee5090605c907c66808664c932b549c"}, + {file = "safetensors-0.4.5.tar.gz", hash = "sha256:d73de19682deabb02524b3d5d1f8b3aaba94c72f1bbfc7911b9b9d5d391c0310"}, +] + +[package.extras] +all = ["safetensors[jax]", "safetensors[numpy]", "safetensors[paddlepaddle]", "safetensors[pinned-tf]", "safetensors[quality]", "safetensors[testing]", "safetensors[torch]"] +dev = ["safetensors[all]"] +jax = ["flax (>=0.6.3)", "jax (>=0.3.25)", "jaxlib (>=0.3.25)", "safetensors[numpy]"] +mlx = ["mlx (>=0.0.9)"] +numpy = ["numpy (>=1.21.6)"] +paddlepaddle = ["paddlepaddle (>=2.4.1)", "safetensors[numpy]"] +pinned-tf = ["safetensors[numpy]", "tensorflow (==2.11.0)"] +quality = ["black (==22.3)", "click (==8.0.4)", "flake8 (>=3.8.3)", "isort (>=5.5.4)"] +tensorflow = ["safetensors[numpy]", "tensorflow (>=2.11.0)"] +testing = ["h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "hypothesis (>=6.70.2)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "safetensors[numpy]", "setuptools-rust (>=1.5.2)"] +torch = ["safetensors[numpy]", "torch (>=1.10)"] + [[package]] name = "sanic" version = "21.12.2" description = "A web server and web framework that's written to go fast. Build fast. Run fast." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -4977,17 +4901,16 @@ uvloop = {version = ">=0.5.3", markers = "sys_platform != \"win32\" and implemen websockets = ">=10.0" [package.extras] -all = ["bandit", "beautifulsoup4", "black", "chardet (>=3.0.0,<4.0.0)", "coverage (==5.3)", "cryptography", "docutils", "flake8", "gunicorn (==20.0.4)", "isort (>=5.0.0)", "m2r2", "mistune (<2.0.0)", "mypy (>=0.901,<0.910)", "pygments", "pytest (==6.2.5)", "pytest-benchmark", "pytest-cov", "pytest-sanic", "pytest-sugar", "sanic-testing (>=0.7.0)", "sphinx (>=2.1.2)", "sphinx-rtd-theme (>=0.4.3)", "towncrier", "tox", "types-ujson", "uvicorn (<0.15.0)"] -dev = ["bandit", "beautifulsoup4", "black", "chardet (>=3.0.0,<4.0.0)", "coverage (==5.3)", "cryptography", "docutils", "flake8", "gunicorn (==20.0.4)", "isort (>=5.0.0)", "mypy (>=0.901,<0.910)", "pygments", "pytest (==6.2.5)", "pytest-benchmark", "pytest-cov", "pytest-sanic", "pytest-sugar", "sanic-testing (>=0.7.0)", "towncrier", "tox", "types-ujson", "uvicorn (<0.15.0)"] +all = ["bandit", "beautifulsoup4", "black", "chardet (==3.*)", "coverage (==5.3)", "cryptography", "docutils", "flake8", "gunicorn (==20.0.4)", "isort (>=5.0.0)", "m2r2", "mistune (<2.0.0)", "mypy (>=0.901,<0.910)", "pygments", "pytest (==6.2.5)", "pytest-benchmark", "pytest-cov", "pytest-sanic", "pytest-sugar", "sanic-testing (>=0.7.0)", "sphinx (>=2.1.2)", "sphinx-rtd-theme (>=0.4.3)", "towncrier", "tox", "types-ujson", "uvicorn (<0.15.0)"] +dev = ["bandit", "beautifulsoup4", "black", "chardet (==3.*)", "coverage (==5.3)", "cryptography", "docutils", "flake8", "gunicorn (==20.0.4)", "isort (>=5.0.0)", "mypy (>=0.901,<0.910)", "pygments", "pytest (==6.2.5)", "pytest-benchmark", "pytest-cov", "pytest-sanic", "pytest-sugar", "sanic-testing (>=0.7.0)", "towncrier", "tox", "types-ujson", "uvicorn (<0.15.0)"] docs = ["docutils", "m2r2", "mistune (<2.0.0)", "pygments", "sphinx (>=2.1.2)", "sphinx-rtd-theme (>=0.4.3)"] ext = ["sanic-ext"] -test = ["bandit", "beautifulsoup4", "black", "chardet (>=3.0.0,<4.0.0)", "coverage (==5.3)", "docutils", "flake8", "gunicorn (==20.0.4)", "isort (>=5.0.0)", "mypy (>=0.901,<0.910)", "pygments", "pytest (==6.2.5)", "pytest-benchmark", "pytest-cov", "pytest-sanic", "pytest-sugar", "sanic-testing (>=0.7.0)", "types-ujson", "uvicorn (<0.15.0)"] +test = ["bandit", "beautifulsoup4", "black", "chardet (==3.*)", "coverage (==5.3)", "docutils", "flake8", "gunicorn (==20.0.4)", "isort (>=5.0.0)", "mypy (>=0.901,<0.910)", "pygments", "pytest (==6.2.5)", "pytest-benchmark", "pytest-cov", "pytest-sanic", "pytest-sugar", "sanic-testing (>=0.7.0)", "types-ujson", "uvicorn (<0.15.0)"] [[package]] name = "sanic-cors" version = "2.0.1" description = "A Sanic extension adding a decorator for CORS support. Based on flask-cors by Cory Dolphin." -category = "main" optional = false python-versions = "*" files = [ @@ -5002,7 +4925,6 @@ sanic = ">=21.9.3" name = "sanic-jwt" version = "1.8.0" description = "JWT oauth flow for Sanic" -category = "main" optional = false python-versions = "*" files = [ @@ -5021,7 +4943,6 @@ docs = ["Jinja2 (<3.1)", "Sphinx"] name = "sanic-routing" version = "0.7.2" description = "Core routing component for Sanic" -category = "main" optional = false python-versions = "*" files = [ @@ -5033,7 +4954,6 @@ files = [ name = "sanic-testing" version = "22.6.0" description = "Core testing clients for Sanic" -category = "dev" optional = false python-versions = "*" files = [ @@ -5048,7 +4968,6 @@ httpx = ">=0.18,<0.24" name = "scikit-learn" version = "1.1.3" description = "A set of python modules for machine learning and data mining" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -5091,7 +5010,6 @@ tests = ["black (>=22.3.0)", "flake8 (>=3.8.2)", "matplotlib (>=3.1.2)", "mypy ( name = "scipy" version = "1.10.1" description = "Fundamental algorithms for scientific computing in Python" -category = "main" optional = false python-versions = "<3.12,>=3.8" files = [ @@ -5130,7 +5048,6 @@ test = ["asv", "gmpy2", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeo name = "sentencepiece" version = "0.1.99" description = "SentencePiece python wrapper" -category = "main" optional = true python-versions = "*" files = [ @@ -5185,7 +5102,6 @@ files = [ name = "sentinels" version = "1.0.0" description = "Various objects to denote special meanings in python" -category = "dev" optional = false python-versions = "*" files = [ @@ -5196,7 +5112,6 @@ files = [ name = "sentry-sdk" version = "1.14.0" description = "Python client for Sentry (https://sentry.io)" -category = "main" optional = false python-versions = "*" files = [ @@ -5233,26 +5148,23 @@ tornado = ["tornado (>=5)"] [[package]] name = "setuptools" -version = "68.0.0" +version = "70.3.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "setuptools-68.0.0-py3-none-any.whl", hash = "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f"}, - {file = "setuptools-68.0.0.tar.gz", hash = "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235"}, + {file = "setuptools-70.3.0-py3-none-any.whl", hash = "sha256:fe384da74336c398e0d956d1cae0669bc02eed936cdb1d49b57de1990dc11ffc"}, + {file = "setuptools-70.3.0.tar.gz", hash = "sha256:f171bab1dfbc86b132997f26a119f6056a57950d058587841a0082e8830f9dc5"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.10.0)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -5264,7 +5176,6 @@ files = [ name = "sklearn-crfsuite" version = "0.3.6" description = "CRFsuite (python-crfsuite) wrapper which provides interface simlar to scikit-learn" -category = "main" optional = false python-versions = "*" files = [ @@ -5278,11 +5189,32 @@ six = "*" tabulate = "*" tqdm = ">=2.0" +[[package]] +name = "skops" +version = "0.9.0" +description = "A set of tools to push scikit-learn based models to and pull from Hugging Face Hub" +optional = false +python-versions = ">=3.8" +files = [ + {file = "skops-0.9.0-py3-none-any.whl", hash = "sha256:05645199bf6976e1f6dbba4a0704799cd5d2fcef18a98b069b4c84744e1a80a1"}, + {file = "skops-0.9.0.tar.gz", hash = "sha256:3e39333d65f26d5863ad44db5001b4cfe6a29642274ac37af54fb834813aee3f"}, +] + +[package.dependencies] +huggingface-hub = ">=0.17.0" +packaging = ">=17.0" +scikit-learn = ">=0.24" +tabulate = ">=0.8.8" + +[package.extras] +docs = ["fairlearn (>=0.7.0)", "matplotlib (>=3.3)", "numpydoc (>=1.0.0)", "pandas (>=1)", "scikit-learn-intelex (>=2021.7.1)", "sphinx (>=3.2.0)", "sphinx-gallery (>=0.7.0)", "sphinx-issues (>=1.2.0)", "sphinx-prompt (>=1.3.0)", "sphinx-rtd-theme (>=1)"] +rich = ["rich (>=12)"] +tests = ["catboost (>=1.0)", "fairlearn (>=0.7.0)", "flake8 (>=3.8.2)", "flaky (>=3.7.0)", "lightgbm (>=3)", "matplotlib (>=3.3)", "pandas (>=1)", "pytest (>=5.0.1)", "pytest-cov (>=2.9.0)", "quantile-forest (>=1.0.0)", "rich (>=12)", "types-requests (>=2.28.5)", "xgboost (>=1.6)"] + [[package]] name = "slack-sdk" version = "3.21.3" description = "The Slack API Platform SDK for Python" -category = "main" optional = false python-versions = ">=3.6.0" files = [ @@ -5298,7 +5230,6 @@ testing = ["Flask (>=1,<2)", "Flask-Sockets (>=0.2,<1)", "Jinja2 (==3.0.3)", "We name = "smart-open" version = "6.3.0" description = "Utils for streaming large files (S3, HDFS, GCS, Azure Blob Storage, gzip, bz2...)" -category = "main" optional = true python-versions = ">=3.6,<4.0" files = [ @@ -5320,7 +5251,6 @@ webhdfs = ["requests"] name = "smmap" version = "5.0.0" description = "A pure Python implementation of a sliding window memory map manager" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -5332,7 +5262,6 @@ files = [ name = "sniffio" version = "1.3.0" description = "Sniff out which async library your code is running under" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -5344,7 +5273,6 @@ files = [ name = "sortedcontainers" version = "2.4.0" description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" -category = "dev" optional = false python-versions = "*" files = [ @@ -5356,7 +5284,6 @@ files = [ name = "spacy" version = "3.4.4" description = "Industrial-strength Natural Language Processing (NLP) in Python" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -5443,7 +5370,6 @@ transformers = ["spacy-transformers (>=1.1.2,<1.2.0)"] name = "spacy" version = "3.5.4" description = "Industrial-strength Natural Language Processing (NLP) in Python" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -5530,7 +5456,6 @@ transformers = ["spacy-transformers (>=1.1.2,<1.3.0)"] name = "spacy-legacy" version = "3.0.12" description = "Legacy registered functions for spaCy backwards compatibility" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -5542,7 +5467,6 @@ files = [ name = "spacy-loggers" version = "1.0.4" description = "Logging utilities for SpaCy" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -5554,7 +5478,6 @@ files = [ name = "sqlalchemy" version = "1.4.49" description = "Database Abstraction Library" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -5562,6 +5485,7 @@ files = [ {file = "SQLAlchemy-1.4.49-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:03db81b89fe7ef3857b4a00b63dedd632d6183d4ea5a31c5d8a92e000a41fc71"}, {file = "SQLAlchemy-1.4.49-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:95b9df9afd680b7a3b13b38adf6e3a38995da5e162cc7524ef08e3be4e5ed3e1"}, {file = "SQLAlchemy-1.4.49-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a63e43bf3f668c11bb0444ce6e809c1227b8f067ca1068898f3008a273f52b09"}, + {file = "SQLAlchemy-1.4.49-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca46de16650d143a928d10842939dab208e8d8c3a9a8757600cae9b7c579c5cd"}, {file = "SQLAlchemy-1.4.49-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f835c050ebaa4e48b18403bed2c0fda986525896efd76c245bdd4db995e51a4c"}, {file = "SQLAlchemy-1.4.49-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c21b172dfb22e0db303ff6419451f0cac891d2e911bb9fbf8003d717f1bcf91"}, {file = "SQLAlchemy-1.4.49-cp310-cp310-win32.whl", hash = "sha256:5fb1ebdfc8373b5a291485757bd6431de8d7ed42c27439f543c81f6c8febd729"}, @@ -5571,26 +5495,35 @@ files = [ {file = "SQLAlchemy-1.4.49-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5debe7d49b8acf1f3035317e63d9ec8d5e4d904c6e75a2a9246a119f5f2fdf3d"}, {file = "SQLAlchemy-1.4.49-cp311-cp311-win32.whl", hash = "sha256:82b08e82da3756765c2e75f327b9bf6b0f043c9c3925fb95fb51e1567fa4ee87"}, {file = "SQLAlchemy-1.4.49-cp311-cp311-win_amd64.whl", hash = "sha256:171e04eeb5d1c0d96a544caf982621a1711d078dbc5c96f11d6469169bd003f1"}, + {file = "SQLAlchemy-1.4.49-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f23755c384c2969ca2f7667a83f7c5648fcf8b62a3f2bbd883d805454964a800"}, + {file = "SQLAlchemy-1.4.49-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8396e896e08e37032e87e7fbf4a15f431aa878c286dc7f79e616c2feacdb366c"}, + {file = "SQLAlchemy-1.4.49-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66da9627cfcc43bbdebd47bfe0145bb662041472393c03b7802253993b6b7c90"}, + {file = "SQLAlchemy-1.4.49-cp312-cp312-win32.whl", hash = "sha256:9a06e046ffeb8a484279e54bda0a5abfd9675f594a2e38ef3133d7e4d75b6214"}, + {file = "SQLAlchemy-1.4.49-cp312-cp312-win_amd64.whl", hash = "sha256:7cf8b90ad84ad3a45098b1c9f56f2b161601e4670827d6b892ea0e884569bd1d"}, {file = "SQLAlchemy-1.4.49-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:36e58f8c4fe43984384e3fbe6341ac99b6b4e083de2fe838f0fdb91cebe9e9cb"}, {file = "SQLAlchemy-1.4.49-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b31e67ff419013f99ad6f8fc73ee19ea31585e1e9fe773744c0f3ce58c039c30"}, + {file = "SQLAlchemy-1.4.49-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebc22807a7e161c0d8f3da34018ab7c97ef6223578fcdd99b1d3e7ed1100a5db"}, {file = "SQLAlchemy-1.4.49-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c14b29d9e1529f99efd550cd04dbb6db6ba5d690abb96d52de2bff4ed518bc95"}, {file = "SQLAlchemy-1.4.49-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c40f3470e084d31247aea228aa1c39bbc0904c2b9ccbf5d3cfa2ea2dac06f26d"}, {file = "SQLAlchemy-1.4.49-cp36-cp36m-win32.whl", hash = "sha256:706bfa02157b97c136547c406f263e4c6274a7b061b3eb9742915dd774bbc264"}, {file = "SQLAlchemy-1.4.49-cp36-cp36m-win_amd64.whl", hash = "sha256:a7f7b5c07ae5c0cfd24c2db86071fb2a3d947da7bd487e359cc91e67ac1c6d2e"}, {file = "SQLAlchemy-1.4.49-cp37-cp37m-macosx_11_0_x86_64.whl", hash = "sha256:4afbbf5ef41ac18e02c8dc1f86c04b22b7a2125f2a030e25bbb4aff31abb224b"}, {file = "SQLAlchemy-1.4.49-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:24e300c0c2147484a002b175f4e1361f102e82c345bf263242f0449672a4bccf"}, + {file = "SQLAlchemy-1.4.49-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:393cd06c3b00b57f5421e2133e088df9cabcececcea180327e43b937b5a7caa5"}, {file = "SQLAlchemy-1.4.49-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:201de072b818f8ad55c80d18d1a788729cccf9be6d9dc3b9d8613b053cd4836d"}, {file = "SQLAlchemy-1.4.49-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7653ed6817c710d0c95558232aba799307d14ae084cc9b1f4c389157ec50df5c"}, {file = "SQLAlchemy-1.4.49-cp37-cp37m-win32.whl", hash = "sha256:647e0b309cb4512b1f1b78471fdaf72921b6fa6e750b9f891e09c6e2f0e5326f"}, {file = "SQLAlchemy-1.4.49-cp37-cp37m-win_amd64.whl", hash = "sha256:ab73ed1a05ff539afc4a7f8cf371764cdf79768ecb7d2ec691e3ff89abbc541e"}, {file = "SQLAlchemy-1.4.49-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:37ce517c011560d68f1ffb28af65d7e06f873f191eb3a73af5671e9c3fada08a"}, {file = "SQLAlchemy-1.4.49-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1878ce508edea4a879015ab5215546c444233881301e97ca16fe251e89f1c55"}, + {file = "SQLAlchemy-1.4.49-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95ab792ca493891d7a45a077e35b418f68435efb3e1706cb8155e20e86a9013c"}, {file = "SQLAlchemy-1.4.49-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0e8e608983e6f85d0852ca61f97e521b62e67969e6e640fe6c6b575d4db68557"}, {file = "SQLAlchemy-1.4.49-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ccf956da45290df6e809ea12c54c02ace7f8ff4d765d6d3dfb3655ee876ce58d"}, {file = "SQLAlchemy-1.4.49-cp38-cp38-win32.whl", hash = "sha256:f167c8175ab908ce48bd6550679cc6ea20ae169379e73c7720a28f89e53aa532"}, {file = "SQLAlchemy-1.4.49-cp38-cp38-win_amd64.whl", hash = "sha256:45806315aae81a0c202752558f0df52b42d11dd7ba0097bf71e253b4215f34f4"}, {file = "SQLAlchemy-1.4.49-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:b6d0c4b15d65087738a6e22e0ff461b407533ff65a73b818089efc8eb2b3e1de"}, {file = "SQLAlchemy-1.4.49-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a843e34abfd4c797018fd8d00ffffa99fd5184c421f190b6ca99def4087689bd"}, + {file = "SQLAlchemy-1.4.49-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:738d7321212941ab19ba2acf02a68b8ee64987b248ffa2101630e8fccb549e0d"}, {file = "SQLAlchemy-1.4.49-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1c890421651b45a681181301b3497e4d57c0d01dc001e10438a40e9a9c25ee77"}, {file = "SQLAlchemy-1.4.49-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d26f280b8f0a8f497bc10573849ad6dc62e671d2468826e5c748d04ed9e670d5"}, {file = "SQLAlchemy-1.4.49-cp39-cp39-win32.whl", hash = "sha256:ec2268de67f73b43320383947e74700e95c6770d0c68c4e615e9897e46296294"}, @@ -5599,7 +5532,7 @@ files = [ ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and platform_machine == \"aarch64\" or python_version >= \"3\" and platform_machine == \"ppc64le\" or python_version >= \"3\" and platform_machine == \"x86_64\" or python_version >= \"3\" and platform_machine == \"amd64\" or python_version >= \"3\" and platform_machine == \"AMD64\" or python_version >= \"3\" and platform_machine == \"win32\" or python_version >= \"3\" and platform_machine == \"WIN32\""} +greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} [package.extras] aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] @@ -5626,7 +5559,6 @@ sqlcipher = ["sqlcipher3-binary"] name = "srsly" version = "2.4.6" description = "Modern high-performance serialization utilities for Python" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -5667,7 +5599,6 @@ catalogue = ">=2.0.3,<2.1.0" name = "stevedore" version = "5.1.0" description = "Manage dynamic plugins for Python applications" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -5682,7 +5613,6 @@ pbr = ">=2.0.0,<2.1.0 || >2.1.0" name = "structlog" version = "23.1.0" description = "Structured Logging for Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -5700,7 +5630,6 @@ typing = ["mypy", "rich", "twisted"] name = "structlog-sentry" version = "2.0.3" description = "Sentry integration for structlog" -category = "main" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -5716,7 +5645,6 @@ structlog = "*" name = "tabulate" version = "0.9.0" description = "Pretty-print tabular data" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -5731,7 +5659,6 @@ widechars = ["wcwidth"] name = "tarsafe" version = "0.0.4" description = "A safe subclass of the TarFile class for interacting with tar files. Can be used as a direct drop-in replacement for safe usage of extractall()" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -5743,7 +5670,6 @@ files = [ name = "tensorboard" version = "2.12.3" description = "TensorBoard lets you watch Tensors Flow" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -5768,7 +5694,6 @@ wheel = ">=0.26" name = "tensorboard-data-server" version = "0.7.1" description = "Fast data loading for TensorBoard" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -5781,7 +5706,6 @@ files = [ name = "tensorflow" version = "2.12.0" description = "TensorFlow is an open source machine learning framework for everyone." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -5831,7 +5755,6 @@ wrapt = ">=1.11.0,<1.15" name = "tensorflow-cpu-aws" version = "2.12.0" description = "TensorFlow is an open source machine learning framework for everyone." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -5869,7 +5792,6 @@ wrapt = ">=1.11.0,<1.15" name = "tensorflow-estimator" version = "2.12.0" description = "TensorFlow Estimator." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -5880,7 +5802,6 @@ files = [ name = "tensorflow-hub" version = "0.13.0" description = "TensorFlow Hub is a library to foster the publication, discovery, and consumption of reusable parts of machine learning models." -category = "main" optional = false python-versions = "*" files = [ @@ -5899,7 +5820,6 @@ make-nearest-neighbour-index = ["annoy", "apache-beam"] name = "tensorflow-intel" version = "2.12.0" description = "TensorFlow is an open source machine learning framework for everyone." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -5937,7 +5857,6 @@ wrapt = ">=1.11.0,<1.15" name = "tensorflow-io-gcs-filesystem" version = "0.31.0" description = "TensorFlow IO" -category = "main" optional = false python-versions = ">=3.7, <3.12" files = [ @@ -5973,7 +5892,6 @@ tensorflow-rocm = ["tensorflow-rocm (>=2.11.0,<2.12.0)"] name = "tensorflow-io-gcs-filesystem" version = "0.32.0" description = "TensorFlow IO" -category = "main" optional = false python-versions = ">=3.7, <3.12" files = [ @@ -6004,7 +5922,6 @@ tensorflow-rocm = ["tensorflow-rocm (>=2.12.0,<2.13.0)"] name = "tensorflow-macos" version = "2.12.0" description = "TensorFlow is an open source machine learning framework for everyone." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -6046,7 +5963,6 @@ wrapt = ">=1.11.0,<1.15" name = "tensorflow-metal" version = "0.8.0" description = "TensorFlow acceleration for Mac GPUs." -category = "main" optional = true python-versions = "*" files = [ @@ -6068,7 +5984,6 @@ wheel = ">=0.35,<1.0" name = "tensorflow-text" version = "2.12.0" description = "TF.Text is a TensorFlow library of text related ops, modules, and subgraphs." -category = "main" optional = false python-versions = "*" files = [ @@ -6094,7 +6009,6 @@ tests = ["absl-py", "pytest", "tensorflow-datasets (>=3.2.0)"] name = "termcolor" version = "2.3.0" description = "ANSI color formatting for output in terminal" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -6109,7 +6023,6 @@ tests = ["pytest", "pytest-cov"] name = "terminaltables" version = "3.1.10" description = "Generate simple tables in terminals from a nested list of strings." -category = "main" optional = false python-versions = ">=2.6" files = [ @@ -6121,7 +6034,6 @@ files = [ name = "thinc" version = "8.1.10" description = "A refreshing functional take on deep learning, compatible with your favorite libraries" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -6197,7 +6109,6 @@ torch = ["torch (>=1.6.0)"] name = "threadpoolctl" version = "3.1.0" description = "threadpoolctl" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -6207,64 +6118,135 @@ files = [ [[package]] name = "tokenizers" -version = "0.13.3" -description = "Fast and Customizable Tokenizers" -category = "main" +version = "0.15.2" +description = "" optional = true -python-versions = "*" +python-versions = ">=3.7" files = [ - {file = "tokenizers-0.13.3-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:f3835c5be51de8c0a092058a4d4380cb9244fb34681fd0a295fbf0a52a5fdf33"}, - {file = "tokenizers-0.13.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:4ef4c3e821730f2692489e926b184321e887f34fb8a6b80b8096b966ba663d07"}, - {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5fd1a6a25353e9aa762e2aae5a1e63883cad9f4e997c447ec39d071020459bc"}, - {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee0b1b311d65beab83d7a41c56a1e46ab732a9eed4460648e8eb0bd69fc2d059"}, - {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ef4215284df1277dadbcc5e17d4882bda19f770d02348e73523f7e7d8b8d396"}, - {file = "tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4d53976079cff8a033f778fb9adca2d9d69d009c02fa2d71a878b5f3963ed30"}, - {file = "tokenizers-0.13.3-cp310-cp310-win32.whl", hash = "sha256:1f0e3b4c2ea2cd13238ce43548959c118069db7579e5d40ec270ad77da5833ce"}, - {file = "tokenizers-0.13.3-cp310-cp310-win_amd64.whl", hash = "sha256:89649c00d0d7211e8186f7a75dfa1db6996f65edce4b84821817eadcc2d3c79e"}, - {file = "tokenizers-0.13.3-cp311-cp311-macosx_10_11_universal2.whl", hash = "sha256:56b726e0d2bbc9243872b0144515ba684af5b8d8cd112fb83ee1365e26ec74c8"}, - {file = "tokenizers-0.13.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:cc5c022ce692e1f499d745af293ab9ee6f5d92538ed2faf73f9708c89ee59ce6"}, - {file = "tokenizers-0.13.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f55c981ac44ba87c93e847c333e58c12abcbb377a0c2f2ef96e1a266e4184ff2"}, - {file = "tokenizers-0.13.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f247eae99800ef821a91f47c5280e9e9afaeed9980fc444208d5aa6ba69ff148"}, - {file = "tokenizers-0.13.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4b3e3215d048e94f40f1c95802e45dcc37c5b05eb46280fc2ccc8cd351bff839"}, - {file = "tokenizers-0.13.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ba2b0bf01777c9b9bc94b53764d6684554ce98551fec496f71bc5be3a03e98b"}, - {file = "tokenizers-0.13.3-cp311-cp311-win32.whl", hash = "sha256:cc78d77f597d1c458bf0ea7c2a64b6aa06941c7a99cb135b5969b0278824d808"}, - {file = "tokenizers-0.13.3-cp311-cp311-win_amd64.whl", hash = "sha256:ecf182bf59bd541a8876deccf0360f5ae60496fd50b58510048020751cf1724c"}, - {file = "tokenizers-0.13.3-cp37-cp37m-macosx_10_11_x86_64.whl", hash = "sha256:0527dc5436a1f6bf2c0327da3145687d3bcfbeab91fed8458920093de3901b44"}, - {file = "tokenizers-0.13.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07cbb2c307627dc99b44b22ef05ff4473aa7c7cc1fec8f0a8b37d8a64b1a16d2"}, - {file = "tokenizers-0.13.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4560dbdeaae5b7ee0d4e493027e3de6d53c991b5002d7ff95083c99e11dd5ac0"}, - {file = "tokenizers-0.13.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64064bd0322405c9374305ab9b4c07152a1474370327499911937fd4a76d004b"}, - {file = "tokenizers-0.13.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8c6e2ab0f2e3d939ca66aa1d596602105fe33b505cd2854a4c1717f704c51de"}, - {file = "tokenizers-0.13.3-cp37-cp37m-win32.whl", hash = "sha256:6cc29d410768f960db8677221e497226e545eaaea01aa3613fa0fdf2cc96cff4"}, - {file = "tokenizers-0.13.3-cp37-cp37m-win_amd64.whl", hash = "sha256:fc2a7fdf864554a0dacf09d32e17c0caa9afe72baf9dd7ddedc61973bae352d8"}, - {file = "tokenizers-0.13.3-cp38-cp38-macosx_10_11_x86_64.whl", hash = "sha256:8791dedba834c1fc55e5f1521be325ea3dafb381964be20684b92fdac95d79b7"}, - {file = "tokenizers-0.13.3-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:d607a6a13718aeb20507bdf2b96162ead5145bbbfa26788d6b833f98b31b26e1"}, - {file = "tokenizers-0.13.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3791338f809cd1bf8e4fee6b540b36822434d0c6c6bc47162448deee3f77d425"}, - {file = "tokenizers-0.13.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2f35f30e39e6aab8716f07790f646bdc6e4a853816cc49a95ef2a9016bf9ce6"}, - {file = "tokenizers-0.13.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:310204dfed5aa797128b65d63538a9837cbdd15da2a29a77d67eefa489edda26"}, - {file = "tokenizers-0.13.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0f9b92ea052305166559f38498b3b0cae159caea712646648aaa272f7160963"}, - {file = "tokenizers-0.13.3-cp38-cp38-win32.whl", hash = "sha256:9a3fa134896c3c1f0da6e762d15141fbff30d094067c8f1157b9fdca593b5806"}, - {file = "tokenizers-0.13.3-cp38-cp38-win_amd64.whl", hash = "sha256:8e7b0cdeace87fa9e760e6a605e0ae8fc14b7d72e9fc19c578116f7287bb873d"}, - {file = "tokenizers-0.13.3-cp39-cp39-macosx_10_11_x86_64.whl", hash = "sha256:00cee1e0859d55507e693a48fa4aef07060c4bb6bd93d80120e18fea9371c66d"}, - {file = "tokenizers-0.13.3-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:a23ff602d0797cea1d0506ce69b27523b07e70f6dda982ab8cf82402de839088"}, - {file = "tokenizers-0.13.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70ce07445050b537d2696022dafb115307abdffd2a5c106f029490f84501ef97"}, - {file = "tokenizers-0.13.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:280ffe95f50eaaf655b3a1dc7ff1d9cf4777029dbbc3e63a74e65a056594abc3"}, - {file = "tokenizers-0.13.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97acfcec592f7e9de8cadcdcda50a7134423ac8455c0166b28c9ff04d227b371"}, - {file = "tokenizers-0.13.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd7730c98a3010cd4f523465867ff95cd9d6430db46676ce79358f65ae39797b"}, - {file = "tokenizers-0.13.3-cp39-cp39-win32.whl", hash = "sha256:48625a108029cb1ddf42e17a81b5a3230ba6888a70c9dc14e81bc319e812652d"}, - {file = "tokenizers-0.13.3-cp39-cp39-win_amd64.whl", hash = "sha256:bc0a6f1ba036e482db6453571c9e3e60ecd5489980ffd95d11dc9f960483d783"}, - {file = "tokenizers-0.13.3.tar.gz", hash = "sha256:2e546dbb68b623008a5442353137fbb0123d311a6d7ba52f2667c8862a75af2e"}, -] - -[package.extras] -dev = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] -docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] + {file = "tokenizers-0.15.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:52f6130c9cbf70544287575a985bf44ae1bda2da7e8c24e97716080593638012"}, + {file = "tokenizers-0.15.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:054c1cc9c6d68f7ffa4e810b3d5131e0ba511b6e4be34157aa08ee54c2f8d9ee"}, + {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a9b9b070fdad06e347563b88c278995735292ded1132f8657084989a4c84a6d5"}, + {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea621a7eef4b70e1f7a4e84dd989ae3f0eeb50fc8690254eacc08acb623e82f1"}, + {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cf7fd9a5141634fa3aa8d6b7be362e6ae1b4cda60da81388fa533e0b552c98fd"}, + {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:44f2a832cd0825295f7179eaf173381dc45230f9227ec4b44378322d900447c9"}, + {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8b9ec69247a23747669ec4b0ca10f8e3dfb3545d550258129bd62291aabe8605"}, + {file = "tokenizers-0.15.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40b6a4c78da863ff26dbd5ad9a8ecc33d8a8d97b535172601cf00aee9d7ce9ce"}, + {file = "tokenizers-0.15.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:5ab2a4d21dcf76af60e05af8063138849eb1d6553a0d059f6534357bce8ba364"}, + {file = "tokenizers-0.15.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a47acfac7e511f6bbfcf2d3fb8c26979c780a91e06fb5b9a43831b2c0153d024"}, + {file = "tokenizers-0.15.2-cp310-none-win32.whl", hash = "sha256:064ff87bb6acdbd693666de9a4b692add41308a2c0ec0770d6385737117215f2"}, + {file = "tokenizers-0.15.2-cp310-none-win_amd64.whl", hash = "sha256:3b919afe4df7eb6ac7cafd2bd14fb507d3f408db7a68c43117f579c984a73843"}, + {file = "tokenizers-0.15.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:89cd1cb93e4b12ff39bb2d626ad77e35209de9309a71e4d3d4672667b4b256e7"}, + {file = "tokenizers-0.15.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cfed5c64e5be23d7ee0f0e98081a25c2a46b0b77ce99a4f0605b1ec43dd481fa"}, + {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a907d76dcfda37023ba203ab4ceeb21bc5683436ebefbd895a0841fd52f6f6f2"}, + {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20ea60479de6fc7b8ae756b4b097572372d7e4032e2521c1bbf3d90c90a99ff0"}, + {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:48e2b9335be2bc0171df9281385c2ed06a15f5cf121c44094338306ab7b33f2c"}, + {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:112a1dd436d2cc06e6ffdc0b06d55ac019a35a63afd26475205cb4b1bf0bfbff"}, + {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4620cca5c2817177ee8706f860364cc3a8845bc1e291aaf661fb899e5d1c45b0"}, + {file = "tokenizers-0.15.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ccd73a82751c523b3fc31ff8194702e4af4db21dc20e55b30ecc2079c5d43cb7"}, + {file = "tokenizers-0.15.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:107089f135b4ae7817affe6264f8c7a5c5b4fd9a90f9439ed495f54fcea56fb4"}, + {file = "tokenizers-0.15.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0ff110ecc57b7aa4a594396525a3451ad70988e517237fe91c540997c4e50e29"}, + {file = "tokenizers-0.15.2-cp311-none-win32.whl", hash = "sha256:6d76f00f5c32da36c61f41c58346a4fa7f0a61be02f4301fd30ad59834977cc3"}, + {file = "tokenizers-0.15.2-cp311-none-win_amd64.whl", hash = "sha256:cc90102ed17271cf0a1262babe5939e0134b3890345d11a19c3145184b706055"}, + {file = "tokenizers-0.15.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f86593c18d2e6248e72fb91c77d413a815153b8ea4e31f7cd443bdf28e467670"}, + {file = "tokenizers-0.15.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0774bccc6608eca23eb9d620196687c8b2360624619623cf4ba9dc9bd53e8b51"}, + {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d0222c5b7c9b26c0b4822a82f6a7011de0a9d3060e1da176f66274b70f846b98"}, + {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3835738be1de66624fff2f4f6f6684775da4e9c00bde053be7564cbf3545cc66"}, + {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0143e7d9dcd811855c1ce1ab9bf5d96d29bf5e528fd6c7824d0465741e8c10fd"}, + {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db35825f6d54215f6b6009a7ff3eedee0848c99a6271c870d2826fbbedf31a38"}, + {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3f5e64b0389a2be47091d8cc53c87859783b837ea1a06edd9d8e04004df55a5c"}, + {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e0480c452217edd35eca56fafe2029fb4d368b7c0475f8dfa3c5c9c400a7456"}, + {file = "tokenizers-0.15.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a33ab881c8fe70474980577e033d0bc9a27b7ab8272896e500708b212995d834"}, + {file = "tokenizers-0.15.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a308a607ca9de2c64c1b9ba79ec9a403969715a1b8ba5f998a676826f1a7039d"}, + {file = "tokenizers-0.15.2-cp312-none-win32.whl", hash = "sha256:b8fcfa81bcb9447df582c5bc96a031e6df4da2a774b8080d4f02c0c16b42be0b"}, + {file = "tokenizers-0.15.2-cp312-none-win_amd64.whl", hash = "sha256:38d7ab43c6825abfc0b661d95f39c7f8af2449364f01d331f3b51c94dcff7221"}, + {file = "tokenizers-0.15.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:38bfb0204ff3246ca4d5e726e8cc8403bfc931090151e6eede54d0e0cf162ef0"}, + {file = "tokenizers-0.15.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9c861d35e8286a53e06e9e28d030b5a05bcbf5ac9d7229e561e53c352a85b1fc"}, + {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:936bf3842db5b2048eaa53dade907b1160f318e7c90c74bfab86f1e47720bdd6"}, + {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:620beacc3373277700d0e27718aa8b25f7b383eb8001fba94ee00aeea1459d89"}, + {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2735ecbbf37e52db4ea970e539fd2d450d213517b77745114f92867f3fc246eb"}, + {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:473c83c5e2359bb81b0b6fde870b41b2764fcdd36d997485e07e72cc3a62264a"}, + {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:968fa1fb3c27398b28a4eca1cbd1e19355c4d3a6007f7398d48826bbe3a0f728"}, + {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:865c60ae6eaebdde7da66191ee9b7db52e542ed8ee9d2c653b6d190a9351b980"}, + {file = "tokenizers-0.15.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7c0d8b52664ab2d4a8d6686eb5effc68b78608a9008f086a122a7b2996befbab"}, + {file = "tokenizers-0.15.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:f33dfbdec3784093a9aebb3680d1f91336c56d86cc70ddf88708251da1fe9064"}, + {file = "tokenizers-0.15.2-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:d44ba80988ff9424e33e0a49445072ac7029d8c0e1601ad25a0ca5f41ed0c1d6"}, + {file = "tokenizers-0.15.2-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:dce74266919b892f82b1b86025a613956ea0ea62a4843d4c4237be2c5498ed3a"}, + {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0ef06b9707baeb98b316577acb04f4852239d856b93e9ec3a299622f6084e4be"}, + {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c73e2e74bbb07910da0d37c326869f34113137b23eadad3fc00856e6b3d9930c"}, + {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4eeb12daf02a59e29f578a865f55d87cd103ce62bd8a3a5874f8fdeaa82e336b"}, + {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9ba9f6895af58487ca4f54e8a664a322f16c26bbb442effd01087eba391a719e"}, + {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ccec77aa7150e38eec6878a493bf8c263ff1fa8a62404e16c6203c64c1f16a26"}, + {file = "tokenizers-0.15.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3f40604f5042ff210ba82743dda2b6aa3e55aa12df4e9f2378ee01a17e2855e"}, + {file = "tokenizers-0.15.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:5645938a42d78c4885086767c70923abad047163d809c16da75d6b290cb30bbe"}, + {file = "tokenizers-0.15.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:05a77cbfebe28a61ab5c3891f9939cc24798b63fa236d84e5f29f3a85a200c00"}, + {file = "tokenizers-0.15.2-cp37-none-win32.whl", hash = "sha256:361abdc068e8afe9c5b818769a48624687fb6aaed49636ee39bec4e95e1a215b"}, + {file = "tokenizers-0.15.2-cp37-none-win_amd64.whl", hash = "sha256:7ef789f83eb0f9baeb4d09a86cd639c0a5518528f9992f38b28e819df397eb06"}, + {file = "tokenizers-0.15.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:4fe1f74a902bee74a3b25aff180fbfbf4f8b444ab37c4d496af7afd13a784ed2"}, + {file = "tokenizers-0.15.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c4b89038a684f40a6b15d6b09f49650ac64d951ad0f2a3ea9169687bbf2a8ba"}, + {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d05a1b06f986d41aed5f2de464c003004b2df8aaf66f2b7628254bcbfb72a438"}, + {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:508711a108684111ec8af89d3a9e9e08755247eda27d0ba5e3c50e9da1600f6d"}, + {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:daa348f02d15160cb35439098ac96e3a53bacf35885072611cd9e5be7d333daa"}, + {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:494fdbe5932d3416de2a85fc2470b797e6f3226c12845cadf054dd906afd0442"}, + {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c2d60f5246f4da9373f75ff18d64c69cbf60c3bca597290cea01059c336d2470"}, + {file = "tokenizers-0.15.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93268e788825f52de4c7bdcb6ebc1fcd4a5442c02e730faa9b6b08f23ead0e24"}, + {file = "tokenizers-0.15.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6fc7083ab404019fc9acafe78662c192673c1e696bd598d16dc005bd663a5cf9"}, + {file = "tokenizers-0.15.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:41e39b41e5531d6b2122a77532dbea60e171ef87a3820b5a3888daa847df4153"}, + {file = "tokenizers-0.15.2-cp38-none-win32.whl", hash = "sha256:06cd0487b1cbfabefb2cc52fbd6b1f8d4c37799bd6c6e1641281adaa6b2504a7"}, + {file = "tokenizers-0.15.2-cp38-none-win_amd64.whl", hash = "sha256:5179c271aa5de9c71712e31cb5a79e436ecd0d7532a408fa42a8dbfa4bc23fd9"}, + {file = "tokenizers-0.15.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:82f8652a74cc107052328b87ea8b34291c0f55b96d8fb261b3880216a9f9e48e"}, + {file = "tokenizers-0.15.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:02458bee6f5f3139f1ebbb6d042b283af712c0981f5bc50edf771d6b762d5e4f"}, + {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c9a09cd26cca2e1c349f91aa665309ddb48d71636370749414fbf67bc83c5343"}, + {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:158be8ea8554e5ed69acc1ce3fbb23a06060bd4bbb09029431ad6b9a466a7121"}, + {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1ddba9a2b0c8c81633eca0bb2e1aa5b3a15362b1277f1ae64176d0f6eba78ab1"}, + {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3ef5dd1d39797044642dbe53eb2bc56435308432e9c7907728da74c69ee2adca"}, + {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:454c203164e07a860dbeb3b1f4a733be52b0edbb4dd2e5bd75023ffa8b49403a"}, + {file = "tokenizers-0.15.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cf6b7f1d4dc59af960e6ffdc4faffe6460bbfa8dce27a58bf75755ffdb2526d"}, + {file = "tokenizers-0.15.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2ef09bbc16519f6c25d0c7fc0c6a33a6f62923e263c9d7cca4e58b8c61572afb"}, + {file = "tokenizers-0.15.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c9a2ebdd2ad4ec7a68e7615086e633857c85e2f18025bd05d2a4399e6c5f7169"}, + {file = "tokenizers-0.15.2-cp39-none-win32.whl", hash = "sha256:918fbb0eab96fe08e72a8c2b5461e9cce95585d82a58688e7f01c2bd546c79d0"}, + {file = "tokenizers-0.15.2-cp39-none-win_amd64.whl", hash = "sha256:524e60da0135e106b254bd71f0659be9f89d83f006ea9093ce4d1fab498c6d0d"}, + {file = "tokenizers-0.15.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:6a9b648a58281c4672212fab04e60648fde574877d0139cd4b4f93fe28ca8944"}, + {file = "tokenizers-0.15.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:7c7d18b733be6bbca8a55084027f7be428c947ddf871c500ee603e375013ffba"}, + {file = "tokenizers-0.15.2-pp310-pypy310_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:13ca3611de8d9ddfbc4dc39ef54ab1d2d4aaa114ac8727dfdc6a6ec4be017378"}, + {file = "tokenizers-0.15.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:237d1bf3361cf2e6463e6c140628e6406766e8b27274f5fcc62c747ae3c6f094"}, + {file = "tokenizers-0.15.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67a0fe1e49e60c664915e9fb6b0cb19bac082ab1f309188230e4b2920230edb3"}, + {file = "tokenizers-0.15.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:4e022fe65e99230b8fd89ebdfea138c24421f91c1a4f4781a8f5016fd5cdfb4d"}, + {file = "tokenizers-0.15.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d857be2df69763362ac699f8b251a8cd3fac9d21893de129bc788f8baaef2693"}, + {file = "tokenizers-0.15.2-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:708bb3e4283177236309e698da5fcd0879ce8fd37457d7c266d16b550bcbbd18"}, + {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:64c35e09e9899b72a76e762f9854e8750213f67567787d45f37ce06daf57ca78"}, + {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1257f4394be0d3b00de8c9e840ca5601d0a4a8438361ce9c2b05c7d25f6057b"}, + {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02272fe48280e0293a04245ca5d919b2c94a48b408b55e858feae9618138aeda"}, + {file = "tokenizers-0.15.2-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dc3ad9ebc76eabe8b1d7c04d38be884b8f9d60c0cdc09b0aa4e3bcf746de0388"}, + {file = "tokenizers-0.15.2-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:32e16bdeffa7c4f46bf2152172ca511808b952701d13e7c18833c0b73cb5c23f"}, + {file = "tokenizers-0.15.2-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:fb16ba563d59003028b678d2361a27f7e4ae0ab29c7a80690efa20d829c81fdb"}, + {file = "tokenizers-0.15.2-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:2277c36d2d6cdb7876c274547921a42425b6810d38354327dd65a8009acf870c"}, + {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1cf75d32e8d250781940d07f7eece253f2fe9ecdb1dc7ba6e3833fa17b82fcbc"}, + {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1b3b31884dc8e9b21508bb76da80ebf7308fdb947a17affce815665d5c4d028"}, + {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b10122d8d8e30afb43bb1fe21a3619f62c3e2574bff2699cf8af8b0b6c5dc4a3"}, + {file = "tokenizers-0.15.2-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d88b96ff0fe8e91f6ef01ba50b0d71db5017fa4e3b1d99681cec89a85faf7bf7"}, + {file = "tokenizers-0.15.2-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:37aaec5a52e959892870a7c47cef80c53797c0db9149d458460f4f31e2fb250e"}, + {file = "tokenizers-0.15.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e2ea752f2b0fe96eb6e2f3adbbf4d72aaa1272079b0dfa1145507bd6a5d537e6"}, + {file = "tokenizers-0.15.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:4b19a808d8799fda23504a5cd31d2f58e6f52f140380082b352f877017d6342b"}, + {file = "tokenizers-0.15.2-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:64c86e5e068ac8b19204419ed8ca90f9d25db20578f5881e337d203b314f4104"}, + {file = "tokenizers-0.15.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de19c4dc503c612847edf833c82e9f73cd79926a384af9d801dcf93f110cea4e"}, + {file = "tokenizers-0.15.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea09acd2fe3324174063d61ad620dec3bcf042b495515f27f638270a7d466e8b"}, + {file = "tokenizers-0.15.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:cf27fd43472e07b57cf420eee1e814549203d56de00b5af8659cb99885472f1f"}, + {file = "tokenizers-0.15.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:7ca22bd897537a0080521445d91a58886c8c04084a6a19e6c78c586e0cfa92a5"}, + {file = "tokenizers-0.15.2.tar.gz", hash = "sha256:e6e9c6e019dd5484be5beafc775ae6c925f4c69a3487040ed09b45e13df2cb91"}, +] + +[package.dependencies] +huggingface_hub = ">=0.16.4,<1.0" + +[package.extras] +dev = ["tokenizers[testing]"] +docs = ["setuptools_rust", "sphinx", "sphinx_rtd_theme"] testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] [[package]] name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -6276,7 +6258,6 @@ files = [ name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -6288,7 +6269,6 @@ files = [ name = "tomli-w" version = "1.0.0" description = "A lil' TOML writer" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -6300,7 +6280,6 @@ files = [ name = "toolz" version = "0.12.0" description = "List processing tools and functional utilities" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -6312,7 +6291,6 @@ files = [ name = "towncrier" version = "22.12.0" description = "Building newsfiles for your project." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -6335,7 +6313,6 @@ dev = ["furo", "packaging", "sphinx (>=5)", "twisted"] name = "tqdm" version = "4.65.0" description = "Fast, Extensible Progress Meter" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -6354,76 +6331,76 @@ telegram = ["requests"] [[package]] name = "transformers" -version = "4.26.0" +version = "4.36.2" description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" -category = "main" optional = true -python-versions = ">=3.7.0" +python-versions = ">=3.8.0" files = [ - {file = "transformers-4.26.0-py3-none-any.whl", hash = "sha256:6a902eee6098d9a737faadf185b8df5a169acc695ebbde5a81b90528f43e665f"}, - {file = "transformers-4.26.0.tar.gz", hash = "sha256:d7859bd83829a3682ca632197ee5c72556e1063d199ab84eec35c4f23b3d73a3"}, + {file = "transformers-4.36.2-py3-none-any.whl", hash = "sha256:462066c4f74ee52516f12890dcc9ec71d1a5e97998db621668455117a54330f6"}, + {file = "transformers-4.36.2.tar.gz", hash = "sha256:d8068e897e47793281501e547d2bbdfc5b8556409c2cb6c3d9e2ca77d4c0b4ec"}, ] [package.dependencies] filelock = "*" -huggingface-hub = ">=0.11.0,<1.0" +huggingface-hub = ">=0.19.3,<1.0" numpy = ">=1.17" packaging = ">=20.0" pyyaml = ">=5.1" regex = "!=2019.12.17" requests = "*" -tokenizers = ">=0.11.1,<0.11.3 || >0.11.3,<0.14" +safetensors = ">=0.3.1" +tokenizers = ">=0.14,<0.19" tqdm = ">=4.27" [package.extras] -accelerate = ["accelerate (>=0.10.0)"] -all = ["Pillow", "accelerate (>=0.10.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8)", "optuna", "phonemizer", "protobuf (<=3.20.2)", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.4,<2.12)", "tensorflow-text", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.7,!=1.12.0)", "torchaudio"] +accelerate = ["accelerate (>=0.21.0)"] +agents = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch (>=1.10,!=1.12.0)"] +all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune] (>=2.7.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision"] audio = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] codecarbon = ["codecarbon (==1.2.0)"] -deepspeed = ["accelerate (>=0.10.0)", "deepspeed (>=0.6.5)"] -deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.10.0)", "beautifulsoup4", "black (==22.3)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.6.5)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "optuna", "parameterized", "protobuf (<=3.20.2)", "psutil", "pytest", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "sentencepiece (>=0.1.91,!=0.1.92)", "timeout-decorator"] -dev = ["GitPython (<3.1.19)", "Pillow", "accelerate (>=0.10.0)", "beautifulsoup4", "black (==22.3)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flake8 (>=3.8.3)", "flax (>=0.4.1)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "optax (>=0.0.8)", "optuna", "parameterized", "phonemizer", "protobuf (<=3.20.2)", "psutil", "pyctcdecode (>=0.4.0)", "pytest", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorflow (>=2.4,<2.12)", "tensorflow-text", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.7,!=1.12.0)", "torchaudio", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)"] -dev-tensorflow = ["GitPython (<3.1.19)", "Pillow", "beautifulsoup4", "black (==22.3)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flake8 (>=3.8.3)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf (<=3.20.2)", "psutil", "pyctcdecode (>=0.4.0)", "pytest", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorflow (>=2.4,<2.12)", "tensorflow-text", "tf2onnx", "timeout-decorator", "tokenizers (>=0.11.1,!=0.11.3,<0.14)"] -dev-torch = ["GitPython (<3.1.19)", "Pillow", "beautifulsoup4", "black (==22.3)", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flake8 (>=3.8.3)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf (<=3.20.2)", "psutil", "pyctcdecode (>=0.4.0)", "pytest", "pytest-timeout", "pytest-xdist", "ray[tune]", "rhoknp (>=1.1.0)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "timeout-decorator", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.7,!=1.12.0)", "torchaudio", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)"] -docs = ["Pillow", "accelerate (>=0.10.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1)", "hf-doc-builder", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8)", "optuna", "phonemizer", "protobuf (<=3.20.2)", "pyctcdecode (>=0.4.0)", "ray[tune]", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.4,<2.12)", "tensorflow-text", "tf2onnx", "timm", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.7,!=1.12.0)", "torchaudio"] +deepspeed = ["accelerate (>=0.21.0)", "deepspeed (>=0.9.3)"] +deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.21.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "optuna", "parameterized", "protobuf", "psutil", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "timeout-decorator"] +dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "decord (==0.6.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +dev-tensorflow = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "nltk", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.14,<0.19)", "urllib3 (<2.0.0)"] +dev-torch = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "hf-doc-builder", "hf-doc-builder (>=0.3.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "librosa", "nltk", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "timeout-decorator", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] +docs = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.21.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "decord (==0.6.0)", "flax (>=0.4.1,<=0.7.0)", "hf-doc-builder", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune] (>=2.7.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timm", "tokenizers (>=0.14,<0.19)", "torch (>=1.10,!=1.12.0)", "torchaudio", "torchvision"] docs-specific = ["hf-doc-builder"] -fairscale = ["fairscale (>0.3)"] -flax = ["flax (>=0.4.1)", "jax (>=0.2.8,!=0.3.2,<=0.3.6)", "jaxlib (>=0.1.65,<=0.3.6)", "optax (>=0.0.8)"] +flax = ["flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "optax (>=0.0.8,<=0.1.4)"] flax-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] ftfy = ["ftfy"] -integrations = ["optuna", "ray[tune]", "sigopt"] -ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0)", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)"] +integrations = ["optuna", "ray[tune] (>=2.7.0)", "sigopt"] +ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)"] modelcreation = ["cookiecutter (==1.7.3)"] -natten = ["natten (>=0.14.4)"] +natten = ["natten (>=0.14.6)"] onnx = ["onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "tf2onnx"] onnxruntime = ["onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)"] optuna = ["optuna"] -quality = ["GitPython (<3.1.19)", "black (==22.3)", "datasets (!=2.5.0)", "flake8 (>=3.8.3)", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)"] -ray = ["ray[tune]"] +quality = ["GitPython (<3.1.19)", "datasets (!=2.5.0)", "hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "ruff (==0.1.5)", "urllib3 (<2.0.0)"] +ray = ["ray[tune] (>=2.7.0)"] retrieval = ["datasets (!=2.5.0)", "faiss-cpu"] sagemaker = ["sagemaker (>=2.31.0)"] -sentencepiece = ["protobuf (<=3.20.2)", "sentencepiece (>=0.1.91,!=0.1.92)"] -serving = ["fastapi", "pydantic", "starlette", "uvicorn"] +sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"] +serving = ["fastapi", "pydantic (<2)", "starlette", "uvicorn"] sigopt = ["sigopt"] sklearn = ["scikit-learn"] speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -testing = ["GitPython (<3.1.19)", "beautifulsoup4", "black (==22.3)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "parameterized", "protobuf (<=3.20.2)", "psutil", "pytest", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "safetensors (>=0.2.1)", "timeout-decorator"] -tf = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow (>=2.4,<2.12)", "tensorflow-text", "tf2onnx"] -tf-cpu = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow-cpu (>=2.4,<2.12)", "tensorflow-text", "tf2onnx"] +testing = ["GitPython (<3.1.19)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "hf-doc-builder (>=0.3.0)", "nltk", "parameterized", "protobuf", "psutil", "pydantic (<2)", "pytest (>=7.2.0)", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.1.5)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "tensorboard", "timeout-decorator"] +tf = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx"] +tf-cpu = ["keras-nlp (>=0.3.1)", "onnxconverter-common", "tensorflow-cpu (>=2.6,<2.16)", "tensorflow-text (<2.16)", "tf2onnx"] tf-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] timm = ["timm"] -tokenizers = ["tokenizers (>=0.11.1,!=0.11.3,<0.14)"] -torch = ["torch (>=1.7,!=1.12.0)"] +tokenizers = ["tokenizers (>=0.14,<0.19)"] +torch = ["accelerate (>=0.21.0)", "torch (>=1.10,!=1.12.0)"] torch-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -torchhub = ["filelock", "huggingface-hub (>=0.11.0,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf (<=3.20.2)", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.11.1,!=0.11.3,<0.14)", "torch (>=1.7,!=1.12.0)", "tqdm (>=4.27)"] -video = ["decord (==0.6.0)"] -vision = ["Pillow"] +torch-vision = ["Pillow (>=10.0.1,<=15.0)", "torchvision"] +torchhub = ["filelock", "huggingface-hub (>=0.19.3,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.14,<0.19)", "torch (>=1.10,!=1.12.0)", "tqdm (>=4.27)"] +video = ["av (==9.2.0)", "decord (==0.6.0)"] +vision = ["Pillow (>=10.0.1,<=15.0)"] [[package]] name = "twilio" version = "8.2.2" description = "Twilio API client and TwiML generator" -category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -6442,7 +6419,6 @@ requests = ">=2.0.0" name = "typer" version = "0.7.0" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -6463,7 +6439,6 @@ test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6. name = "typer" version = "0.9.0" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -6485,7 +6460,6 @@ test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6. name = "types-pyopenssl" version = "23.2.0.1" description = "Typing stubs for pyOpenSSL" -category = "dev" optional = false python-versions = "*" files = [ @@ -6500,7 +6474,6 @@ cryptography = ">=35.0.0" name = "types-python-dateutil" version = "2.8.19.13" description = "Typing stubs for python-dateutil" -category = "dev" optional = false python-versions = "*" files = [ @@ -6512,7 +6485,6 @@ files = [ name = "types-pytz" version = "2022.7.1.2" description = "Typing stubs for pytz" -category = "dev" optional = false python-versions = "*" files = [ @@ -6524,7 +6496,6 @@ files = [ name = "types-redis" version = "4.6.0.2" description = "Typing stubs for redis" -category = "dev" optional = false python-versions = "*" files = [ @@ -6540,7 +6511,6 @@ types-pyOpenSSL = "*" name = "types-requests" version = "2.31.0.1" description = "Typing stubs for requests" -category = "dev" optional = false python-versions = "*" files = [ @@ -6555,7 +6525,6 @@ types-urllib3 = "*" name = "types-setuptools" version = "67.8.0.0" description = "Typing stubs for setuptools" -category = "dev" optional = false python-versions = "*" files = [ @@ -6567,7 +6536,6 @@ files = [ name = "types-toml" version = "0.10.8.6" description = "Typing stubs for toml" -category = "dev" optional = false python-versions = "*" files = [ @@ -6579,7 +6547,6 @@ files = [ name = "types-urllib3" version = "1.26.25.13" description = "Typing stubs for urllib3" -category = "dev" optional = false python-versions = "*" files = [ @@ -6591,7 +6558,6 @@ files = [ name = "typing-extensions" version = "4.7.1" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -6603,7 +6569,6 @@ files = [ name = "typing-utils" version = "0.1.0" description = "utils to inspect Python type annotations" -category = "main" optional = false python-versions = ">=3.6.1" files = [ @@ -6618,7 +6583,6 @@ test = ["pytest"] name = "tzdata" version = "2023.3" description = "Provider of IANA time zone data" -category = "main" optional = false python-versions = ">=2" files = [ @@ -6630,7 +6594,6 @@ files = [ name = "tzlocal" version = "5.0.1" description = "tzinfo object for the local timezone" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -6649,7 +6612,6 @@ devenv = ["black", "check-manifest", "flake8", "pyroma", "pytest (>=4.3)", "pyte name = "ujson" version = "5.8.0" description = "Ultra fast JSON encoder and decoder for Python" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -6720,7 +6682,6 @@ files = [ name = "uritemplate" version = "4.1.1" description = "Implementation of RFC 6570 URI Templates" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -6732,7 +6693,6 @@ files = [ name = "urllib3" version = "1.26.16" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -6749,7 +6709,6 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] name = "uvloop" version = "0.17.0" description = "Fast implementation of asyncio event loop on top of libuv" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -6794,7 +6753,6 @@ test = ["Cython (>=0.29.32,<0.30.0)", "aiohttp", "flake8 (>=3.9.2,<3.10.0)", "my name = "wasabi" version = "0.10.1" description = "A lightweight console printing and formatting toolkit" -category = "main" optional = true python-versions = "*" files = [ @@ -6806,7 +6764,6 @@ files = [ name = "wasabi" version = "1.1.2" description = "A lightweight console printing and formatting toolkit" -category = "main" optional = true python-versions = ">=3.6" files = [ @@ -6818,7 +6775,6 @@ files = [ name = "watchdog" version = "3.0.0" description = "Filesystem events monitoring" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -6858,7 +6814,6 @@ watchmedo = ["PyYAML (>=3.10)"] name = "wcwidth" version = "0.2.6" description = "Measures the displayed width of unicode strings in a terminal" -category = "main" optional = false python-versions = "*" files = [ @@ -6870,7 +6825,6 @@ files = [ name = "webexteamssdk" version = "1.6.1" description = "Community-developed Python SDK for the Webex Teams APIs" -category = "main" optional = false python-versions = "*" files = [ @@ -6888,7 +6842,6 @@ requests-toolbelt = "*" name = "websocket-client" version = "1.6.1" description = "WebSocket client for Python with low level API options" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -6905,7 +6858,6 @@ test = ["websockets"] name = "websockets" version = "10.4" description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -6984,7 +6936,6 @@ files = [ name = "werkzeug" version = "2.3.6" description = "The comprehensive WSGI web application library." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -7002,7 +6953,6 @@ watchdog = ["watchdog (>=2.3)"] name = "wheel" version = "0.40.0" description = "A built-package format for Python" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -7017,7 +6967,6 @@ test = ["pytest (>=6.0.0)"] name = "wrapt" version = "1.14.1" description = "Module for decorators, wrappers and monkey patching." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -7040,6 +6989,16 @@ files = [ {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8ad85f7f4e20964db4daadcab70b47ab05c7c1cf2a7c1e51087bfaa83831854c"}, {file = "wrapt-1.14.1-cp310-cp310-win32.whl", hash = "sha256:a9a52172be0b5aae932bef82a79ec0a0ce87288c7d132946d645eba03f0ad8a8"}, {file = "wrapt-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:6d323e1554b3d22cfc03cd3243b5bb815a51f5249fdcbb86fda4bf62bab9e164"}, + {file = "wrapt-1.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ecee4132c6cd2ce5308e21672015ddfed1ff975ad0ac8d27168ea82e71413f55"}, + {file = "wrapt-1.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2020f391008ef874c6d9e208b24f28e31bcb85ccff4f335f15a3251d222b92d9"}, + {file = "wrapt-1.14.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2feecf86e1f7a86517cab34ae6c2f081fd2d0dac860cb0c0ded96d799d20b335"}, + {file = "wrapt-1.14.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:240b1686f38ae665d1b15475966fe0472f78e71b1b4903c143a842659c8e4cb9"}, + {file = "wrapt-1.14.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9008dad07d71f68487c91e96579c8567c98ca4c3881b9b113bc7b33e9fd78b8"}, + {file = "wrapt-1.14.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6447e9f3ba72f8e2b985a1da758767698efa72723d5b59accefd716e9e8272bf"}, + {file = "wrapt-1.14.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:acae32e13a4153809db37405f5eba5bac5fbe2e2ba61ab227926a22901051c0a"}, + {file = "wrapt-1.14.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49ef582b7a1152ae2766557f0550a9fcbf7bbd76f43fbdc94dd3bf07cc7168be"}, + {file = "wrapt-1.14.1-cp311-cp311-win32.whl", hash = "sha256:358fe87cc899c6bb0ddc185bf3dbfa4ba646f05b1b0b9b5a27c2cb92c2cea204"}, + {file = "wrapt-1.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:26046cd03936ae745a502abf44dac702a5e6880b2b01c29aea8ddf3353b68224"}, {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:43ca3bbbe97af00f49efb06e352eae40434ca9d915906f77def219b88e85d907"}, {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:6b1a564e6cb69922c7fe3a678b9f9a3c54e72b469875aa8018f18b4d1dd1adf3"}, {file = "wrapt-1.14.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3"}, @@ -7091,7 +7050,6 @@ files = [ name = "xmltodict" version = "0.13.0" description = "Makes working with XML feel like you are working with JSON" -category = "dev" optional = false python-versions = ">=3.4" files = [ @@ -7103,7 +7061,6 @@ files = [ name = "yapf" version = "0.40.1" description = "A formatter for Python code." -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -7120,7 +7077,6 @@ tomli = ">=2.0.1" name = "yarl" version = "1.9.2" description = "Yet another URL library" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -7208,7 +7164,6 @@ multidict = ">=4.0" name = "zipp" version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -7231,4 +7186,4 @@ transformers = ["sentencepiece", "transformers"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.11" -content-hash = "71bfb81a213fc85cbf7fe0ac10a9ac16363fd14c18647e1f0ee26b35e73e0747" +content-hash = "c1c51259ab3b886039dcf7eb746a45815d4b8afcaa4bdbe179c891810aee553f" diff --git a/pyproject.toml b/pyproject.toml index c3809092b5c2..c37977e46891 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ exclude = "((.eggs | .git | .pytest_cache | build | dist))" [tool.poetry] name = "rasa" -version = "3.6.13" +version = "3.6.21" description = "Open source machine learning framework to automate text- and voice-based conversations: NLU, dialogue management, connect to Slack, Facebook, and more - Create chatbots and voice assistants" authors = [ "Rasa Technologies GmbH ",] maintainers = [ "Tom Bocklisch ",] @@ -24,6 +24,7 @@ license = "Apache-2.0" [[tool.poetry.source]] name = "internal repository mirroring psycopg binary for macos" url = "https://europe-west3-python.pkg.dev/rasa-releases/psycopg-binary/simple/" +priority = "supplemental" [tool.towncrier] package = "rasa" @@ -119,8 +120,7 @@ sanic-cors = "~2.0.0" sanic-jwt = "^1.6.0" sanic-routing = "^0.7.2" websockets = ">=10.0,<11.0" -cloudpickle = ">=1.2,<2.3" -aiohttp = ">=3.6,!=3.7.4.post0,<3.9" +aiohttp = ">=3.9.0,<3.10" questionary = ">=1.5.1,<1.11.0" prompt-toolkit = "^3.0,<3.0.29" python-socketio = ">=4.4,<6" @@ -132,10 +132,9 @@ psycopg2-binary = ">=2.8.2,<2.10.0" python-dateutil = "~2.8" protobuf = ">=4.23.3,< 4.23.4" tensorflow_hub = "^0.13.0" -setuptools = ">=65.5.1" +setuptools = "~70.3.0" ujson = ">=1.35,<6.0" regex = ">=2020.6,<2022.11" -joblib = ">=0.15.1,<1.3.0" sentry-sdk = ">=0.17.0,<1.15.0" aio-pika = ">=6.7.1,<8.2.4" aiogram = "<2.26" @@ -154,7 +153,9 @@ structlog-sentry = "^2.0.2" dnspython = "2.3.0" wheel = ">=0.38.1" certifi = ">=2023.7.22" -cryptography = ">=41.0.2" +cryptography = ">=41.0.7" +skops = "0.9.0" +safetensors = "~0.4.5" [[tool.poetry.dependencies.tensorflow-io-gcs-filesystem]] version = "==0.31" markers = "sys_platform == 'win32'" @@ -193,7 +194,7 @@ version = ">=1.4.1,<1.7.3" python = "~=3.7.0" [[tool.poetry.dependencies.scipy]] -version = ">=1.10.0" +version = ">=1.10.0,<1.11.0" python = ">=3.8,<3.11" [[tool.poetry.dependencies.scikit-learn]] @@ -284,7 +285,7 @@ version = "~3.2.0" optional = true [tool.poetry.dependencies.transformers] -version = ">=4.13.0, <=4.26.0" +version = "~4.36.2" optional = true [tool.poetry.dependencies.sentencepiece] @@ -313,7 +314,7 @@ pytest-xdist = "^3.2.1" pytest = "^7.1.3" freezegun = "^1.0.0" responses = "^0.22.0" -aioresponses = "^0.7.2" +aioresponses = "^0.7.6" moto = "~=4.1.2" fakeredis = "^2.11.2" mongomock = "^4.1.2" diff --git a/rasa/constants.py b/rasa/constants.py index ef973613bc96..ba883c66b0e0 100644 --- a/rasa/constants.py +++ b/rasa/constants.py @@ -18,7 +18,7 @@ CONFIG_TELEMETRY_ENABLED = "enabled" CONFIG_TELEMETRY_DATE = "date" -MINIMUM_COMPATIBLE_VERSION = "3.5.0" +MINIMUM_COMPATIBLE_VERSION = "3.6.21" GLOBAL_USER_CONFIG_PATH = os.path.expanduser("~/.config/rasa/global.yml") diff --git a/rasa/core/agent.py b/rasa/core/agent.py index bf3d42236e70..47e3360f6a5b 100644 --- a/rasa/core/agent.py +++ b/rasa/core/agent.py @@ -112,53 +112,59 @@ async def _pull_model_and_fingerprint( logger.debug(f"Requesting model from server {model_server.url}...") - try: - params = model_server.combine_parameters() - async with model_server.session.request( - "GET", - model_server.url, - timeout=DEFAULT_REQUEST_TIMEOUT, - headers=headers, - params=params, - ) as resp: - if resp.status in [204, 304]: - logger.debug( - "Model server returned {} status code, " - "indicating that no new model is available. " - "Current fingerprint: {}" - "".format(resp.status, fingerprint) - ) - return None - elif resp.status == 404: - logger.debug( - "Model server could not find a model at the requested " - "endpoint '{}'. It's possible that no model has been " - "trained, or that the requested tag hasn't been " - "assigned.".format(model_server.url) - ) - return None - elif resp.status != 200: - logger.debug( - "Tried to fetch model from server, but server response " - "status code is {}. We'll retry later..." - "".format(resp.status) + async with model_server.session() as session: + try: + params = model_server.combine_parameters() + async with session.request( + "GET", + model_server.url, + timeout=DEFAULT_REQUEST_TIMEOUT, + headers=headers, + params=params, + ) as resp: + + if resp.status in [204, 304]: + logger.debug( + "Model server returned {} status code, " + "indicating that no new model is available. " + "Current fingerprint: {}" + "".format(resp.status, fingerprint) + ) + return None + elif resp.status == 404: + logger.debug( + "Model server could not find a model at the requested " + "endpoint '{}'. It's possible that no model has been " + "trained, or that the requested tag hasn't been " + "assigned.".format(model_server.url) + ) + return None + elif resp.status != 200: + logger.debug( + "Tried to fetch model from server, but server response " + "status code is {}. We'll retry later..." + "".format(resp.status) + ) + return None + + model_path = Path(model_directory) / resp.headers.get( + "filename", "model.tar.gz" ) - return None - model_path = Path(model_directory) / resp.headers.get( - "filename", "model.tar.gz" + with open(model_path, "wb") as file: + file.write(await resp.read()) + + logger.debug("Saved model to '{}'".format(os.path.abspath(model_path))) + + # return the new fingerprint + return resp.headers.get("ETag") + + except aiohttp.ClientError as e: + logger.debug( + "Tried to fetch model from server, but " + "couldn't reach server. We'll retry later... " + "Error: {}.".format(e) ) - with open(model_path, "wb") as file: - file.write(await resp.read()) - logger.debug("Saved model to '{}'".format(os.path.abspath(model_path))) - # return the new fingerprint - return resp.headers.get("ETag") - except aiohttp.ClientError as e: - logger.debug( - "Tried to fetch model from server, but " - "couldn't reach server. We'll retry later... " - "Error: {}.".format(e) - ) - return None + return None async def _run_model_pulling_worker(model_server: EndpointConfig, agent: Agent) -> None: diff --git a/rasa/core/brokers/kafka.py b/rasa/core/brokers/kafka.py index 7183be12746a..66e77c2ca385 100644 --- a/rasa/core/brokers/kafka.py +++ b/rasa/core/brokers/kafka.py @@ -260,9 +260,11 @@ def _publish(self, event: Dict[Text, Any]) -> None: on_delivery=delivery_report, ) - def _close(self) -> None: + async def close(self) -> None: self._cancelled = True self._poll_thread.join() + if self.producer: + self.producer.flush() @rasa.shared.utils.common.lazy_property def rasa_environment(self) -> Optional[Text]: diff --git a/rasa/core/channels/socketio.py b/rasa/core/channels/socketio.py index c268447f020a..52f93dc46bb3 100644 --- a/rasa/core/channels/socketio.py +++ b/rasa/core/channels/socketio.py @@ -233,7 +233,7 @@ async def session_request(sid: Text, data: Optional[Dict]) -> None: if "session_id" not in data or data["session_id"] is None: data["session_id"] = uuid.uuid4().hex if self.session_persistence: - sio.enter_room(sid, data["session_id"]) + await sio.enter_room(sid, data["session_id"]) await sio.emit("session_confirm", data["session_id"], room=sid) logger.debug(f"User {sid} connected to socketIO endpoint.") diff --git a/rasa/core/constants.py b/rasa/core/constants.py index 973e4e7b3a99..40d65c3299bb 100644 --- a/rasa/core/constants.py +++ b/rasa/core/constants.py @@ -24,6 +24,8 @@ DEFAULT_LOCK_LIFETIME = 60 # in seconds +DEFAULT_KEEP_ALIVE_TIMEOUT = 120 # in seconds + BEARER_TOKEN_PREFIX = "Bearer " # The lowest priority is intended to be used by machine learning policies. diff --git a/rasa/core/exporter.py b/rasa/core/exporter.py index 58c567dbdbe2..2961b6b6fc94 100644 --- a/rasa/core/exporter.py +++ b/rasa/core/exporter.py @@ -160,7 +160,7 @@ def _validate_all_requested_ids_exist( self, conversation_ids_in_tracker_store: Set[Text] ) -> None: """Warn user if `self.requested_conversation_ids` contains IDs not found in - `conversation_ids_in_tracker_store` + `conversation_ids_in_tracker_store`. Args: conversation_ids_in_tracker_store: Set of conversation IDs contained in @@ -241,6 +241,12 @@ async def _fetch_events_within_time_range(self) -> AsyncIterator[Dict[Text, Any] continue events = self._get_events_for_conversation_id(_events, conversation_id) + + # the order of events was changed after ATO-2192 + # more context: https://github.com/RasaHQ/rasa/pull/13019 + # we should sort the events by timestamp to keep the order + events.sort(key=lambda x: x["timestamp"]) + # the conversation IDs are needed in the event publishing for event in events: if ( diff --git a/rasa/core/featurizers/single_state_featurizer.py b/rasa/core/featurizers/single_state_featurizer.py index 7d8c504084c1..0a6c921491a4 100644 --- a/rasa/core/featurizers/single_state_featurizer.py +++ b/rasa/core/featurizers/single_state_featurizer.py @@ -1,7 +1,8 @@ import logging +from typing import List, Optional, Dict, Text, Set, Any + import numpy as np import scipy.sparse -from typing import List, Optional, Dict, Text, Set, Any from rasa.core.featurizers.precomputation import MessageContainerForCoreFeaturization from rasa.nlu.extractors.extractor import EntityTagSpec @@ -362,6 +363,26 @@ def encode_all_labels( for action in domain.action_names_or_texts ] + def to_dict(self) -> Dict[str, Any]: + return { + "action_texts": self.action_texts, + "entity_tag_specs": self.entity_tag_specs, + "feature_states": self._default_feature_states, + } + + @classmethod + def create_from_dict( + cls, data: Dict[str, Any] + ) -> Optional["SingleStateFeaturizer"]: + if not data: + return None + + featurizer = SingleStateFeaturizer() + featurizer.action_texts = data["action_texts"] + featurizer._default_feature_states = data["feature_states"] + featurizer.entity_tag_specs = data["entity_tag_specs"] + return featurizer + class IntentTokenizerSingleStateFeaturizer(SingleStateFeaturizer): """A SingleStateFeaturizer for use with policies that predict intent labels.""" diff --git a/rasa/core/featurizers/tracker_featurizers.py b/rasa/core/featurizers/tracker_featurizers.py index 42df6e4e1187..9c6dbca92d47 100644 --- a/rasa/core/featurizers/tracker_featurizers.py +++ b/rasa/core/featurizers/tracker_featurizers.py @@ -1,11 +1,9 @@ from __future__ import annotations -from pathlib import Path -from collections import defaultdict -from abc import abstractmethod -import jsonpickle -import logging -from tqdm import tqdm +import logging +from abc import abstractmethod +from collections import defaultdict +from pathlib import Path from typing import ( Tuple, List, @@ -18,25 +16,30 @@ Set, DefaultDict, cast, + Type, + Callable, + ClassVar, ) + import numpy as np +from tqdm import tqdm -from rasa.core.featurizers.single_state_featurizer import SingleStateFeaturizer -from rasa.core.featurizers.precomputation import MessageContainerForCoreFeaturization -from rasa.core.exceptions import InvalidTrackerFeaturizerUsageError import rasa.shared.core.trackers import rasa.shared.utils.io -from rasa.shared.nlu.constants import TEXT, INTENT, ENTITIES, ACTION_NAME -from rasa.shared.nlu.training_data.features import Features -from rasa.shared.core.trackers import DialogueStateTracker -from rasa.shared.core.domain import State, Domain -from rasa.shared.core.events import Event, ActionExecuted, UserUttered +from rasa.core.exceptions import InvalidTrackerFeaturizerUsageError +from rasa.core.featurizers.precomputation import MessageContainerForCoreFeaturization +from rasa.core.featurizers.single_state_featurizer import SingleStateFeaturizer from rasa.shared.core.constants import ( USER, ACTION_UNLIKELY_INTENT_NAME, PREVIOUS_ACTION, ) +from rasa.shared.core.domain import State, Domain +from rasa.shared.core.events import Event, ActionExecuted, UserUttered +from rasa.shared.core.trackers import DialogueStateTracker from rasa.shared.exceptions import RasaException +from rasa.shared.nlu.constants import TEXT, INTENT, ENTITIES, ACTION_NAME +from rasa.shared.nlu.training_data.features import Features from rasa.utils.tensorflow.constants import LABEL_PAD_ID from rasa.utils.tensorflow.model_data import ragged_array_to_ndarray @@ -64,6 +67,10 @@ def __str__(self) -> Text: class TrackerFeaturizer: """Base class for actual tracker featurizers.""" + # Class registry to store all subclasses + _registry: ClassVar[Dict[str, Type["TrackerFeaturizer"]]] = {} + _featurizer_type: str = "TrackerFeaturizer" + def __init__( self, state_featurizer: Optional[SingleStateFeaturizer] = None ) -> None: @@ -74,6 +81,36 @@ def __init__( """ self.state_featurizer = state_featurizer + @classmethod + def register(cls, featurizer_type: str) -> Callable: + """Decorator to register featurizer subclasses.""" + + def wrapper(subclass: Type["TrackerFeaturizer"]) -> Type["TrackerFeaturizer"]: + cls._registry[featurizer_type] = subclass + # Store the type identifier in the class for serialization + subclass._featurizer_type = featurizer_type + return subclass + + return wrapper + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "TrackerFeaturizer": + """Create featurizer instance from dictionary.""" + featurizer_type = data.pop("type") + + if featurizer_type not in cls._registry: + raise ValueError(f"Unknown featurizer type: {featurizer_type}") + + # Get the correct subclass and instantiate it + subclass = cls._registry[featurizer_type] + return subclass.create_from_dict(data) + + @classmethod + @abstractmethod + def create_from_dict(cls, data: Dict[str, Any]) -> "TrackerFeaturizer": + """Each subclass must implement its own creation from dict method.""" + pass + @staticmethod def _create_states( tracker: DialogueStateTracker, @@ -465,9 +502,7 @@ def persist(self, path: Union[Text, Path]) -> None: self.state_featurizer.entity_tag_specs = [] # noinspection PyTypeChecker - rasa.shared.utils.io.write_text_file( - str(jsonpickle.encode(self)), featurizer_file - ) + rasa.shared.utils.io.dump_obj_as_json_to_file(featurizer_file, self.to_dict()) @staticmethod def load(path: Union[Text, Path]) -> Optional[TrackerFeaturizer]: @@ -481,7 +516,17 @@ def load(path: Union[Text, Path]) -> Optional[TrackerFeaturizer]: """ featurizer_file = Path(path) / FEATURIZER_FILE if featurizer_file.is_file(): - return jsonpickle.decode(rasa.shared.utils.io.read_file(featurizer_file)) + data = rasa.shared.utils.io.read_json_file(featurizer_file) + + if "type" not in data: + logger.error( + f"Couldn't load featurizer for policy. " + f"File '{featurizer_file}' does not contain all " + f"necessary information. 'type' is missing." + ) + return None + + return TrackerFeaturizer.from_dict(data) logger.error( f"Couldn't load featurizer for policy. " @@ -508,7 +553,16 @@ def _remove_action_unlikely_intent_from_events(events: List[Event]) -> List[Even ) ] + def to_dict(self) -> Dict[str, Any]: + return { + "type": self.__class__._featurizer_type, + "state_featurizer": ( + self.state_featurizer.to_dict() if self.state_featurizer else None + ), + } + +@TrackerFeaturizer.register("FullDialogueTrackerFeaturizer") class FullDialogueTrackerFeaturizer(TrackerFeaturizer): """Creates full dialogue training data for time distributed architectures. @@ -646,7 +700,20 @@ def prediction_states( return trackers_as_states + def to_dict(self) -> Dict[str, Any]: + return super().to_dict() + @classmethod + def create_from_dict(cls, data: Dict[str, Any]) -> "FullDialogueTrackerFeaturizer": + state_featurizer = SingleStateFeaturizer.create_from_dict( + data["state_featurizer"] + ) + return cls( + state_featurizer, + ) + + +@TrackerFeaturizer.register("MaxHistoryTrackerFeaturizer") class MaxHistoryTrackerFeaturizer(TrackerFeaturizer): """Truncates the tracker history into `max_history` long sequences. @@ -887,7 +954,25 @@ def prediction_states( return trackers_as_states + def to_dict(self) -> Dict[str, Any]: + data = super().to_dict() + data.update( + { + "remove_duplicates": self.remove_duplicates, + "max_history": self.max_history, + } + ) + return data + + @classmethod + def create_from_dict(cls, data: Dict[str, Any]) -> "MaxHistoryTrackerFeaturizer": + state_featurizer = SingleStateFeaturizer.create_from_dict( + data["state_featurizer"] + ) + return cls(state_featurizer, data["max_history"], data["remove_duplicates"]) + +@TrackerFeaturizer.register("IntentMaxHistoryTrackerFeaturizer") class IntentMaxHistoryTrackerFeaturizer(MaxHistoryTrackerFeaturizer): """Truncates the tracker history into `max_history` long sequences. @@ -1166,6 +1251,18 @@ def prediction_states( return trackers_as_states + def to_dict(self) -> Dict[str, Any]: + return super().to_dict() + + @classmethod + def create_from_dict( + cls, data: Dict[str, Any] + ) -> "IntentMaxHistoryTrackerFeaturizer": + state_featurizer = SingleStateFeaturizer.create_from_dict( + data["state_featurizer"] + ) + return cls(state_featurizer, data["max_history"], data["remove_duplicates"]) + def _is_prev_action_unlikely_intent_in_state(state: State) -> bool: prev_action_name = state.get(PREVIOUS_ACTION, {}).get(ACTION_NAME) diff --git a/rasa/core/policies/ted_policy.py b/rasa/core/policies/ted_policy.py index c5f895e3ce64..af96af627de6 100644 --- a/rasa/core/policies/ted_policy.py +++ b/rasa/core/policies/ted_policy.py @@ -1,15 +1,15 @@ from __future__ import annotations -import logging -from rasa.engine.recipes.default_recipe import DefaultV1Recipe +import logging from pathlib import Path from collections import defaultdict import contextlib +from typing import Any, List, Optional, Text, Dict, Tuple, Union, Type import numpy as np import tensorflow as tf -from typing import Any, List, Optional, Text, Dict, Tuple, Union, Type +from rasa.engine.recipes.default_recipe import DefaultV1Recipe from rasa.engine.graph import ExecutionContext from rasa.engine.storage.resource import Resource from rasa.engine.storage.storage import ModelStorage @@ -49,18 +49,22 @@ from rasa.shared.core.events import EntitiesAdded, Event from rasa.shared.core.domain import Domain from rasa.shared.nlu.training_data.message import Message -from rasa.shared.nlu.training_data.features import Features +from rasa.shared.nlu.training_data.features import ( + Features, + save_features, + load_features, +) import rasa.shared.utils.io import rasa.utils.io from rasa.utils import train_utils -from rasa.utils.tensorflow.models import RasaModel, TransformerRasaModel -from rasa.utils.tensorflow import rasa_layers -from rasa.utils.tensorflow.model_data import ( - RasaModelData, - FeatureSignature, +from rasa.utils.tensorflow.feature_array import ( FeatureArray, - Data, + serialize_nested_feature_arrays, + deserialize_nested_feature_arrays, ) +from rasa.utils.tensorflow.models import RasaModel, TransformerRasaModel +from rasa.utils.tensorflow import rasa_layers +from rasa.utils.tensorflow.model_data import RasaModelData, FeatureSignature, Data from rasa.utils.tensorflow.model_data_utils import convert_to_data_format from rasa.utils.tensorflow.constants import ( LABEL, @@ -961,22 +965,32 @@ def persist_model_utilities(self, model_path: Path) -> None: model_path: Path where model is to be persisted """ model_filename = self._metadata_filename() - rasa.utils.io.json_pickle( - model_path / f"{model_filename}.priority.pkl", self.priority - ) - rasa.utils.io.pickle_dump( - model_path / f"{model_filename}.meta.pkl", self.config + rasa.shared.utils.io.dump_obj_as_json_to_file( + model_path / f"{model_filename}.priority.json", self.priority ) - rasa.utils.io.pickle_dump( - model_path / f"{model_filename}.data_example.pkl", self.data_example + rasa.shared.utils.io.dump_obj_as_json_to_file( + model_path / f"{model_filename}.meta.json", self.config ) - rasa.utils.io.pickle_dump( - model_path / f"{model_filename}.fake_features.pkl", self.fake_features + # save data example + serialize_nested_feature_arrays( + self.data_example, + str(model_path / f"{model_filename}.data_example.st"), + str(model_path / f"{model_filename}.data_example_metadata.json"), ) - rasa.utils.io.pickle_dump( - model_path / f"{model_filename}.label_data.pkl", + # save label data + serialize_nested_feature_arrays( dict(self._label_data.data) if self._label_data is not None else {}, + str(model_path / f"{model_filename}.label_data.st"), + str(model_path / f"{model_filename}.label_data_metadata.json"), + ) + # save fake features + metadata = save_features( + self.fake_features, str(model_path / f"{model_filename}.fake_features.st") + ) + rasa.shared.utils.io.dump_obj_as_json_to_file( + model_path / f"{model_filename}.fake_features_metadata.json", metadata ) + entity_tag_specs = ( [tag_spec._asdict() for tag_spec in self._entity_tag_specs] if self._entity_tag_specs @@ -994,18 +1008,29 @@ def _load_model_utilities(cls, model_path: Path) -> Dict[Text, Any]: model_path: Path where model is to be persisted. """ tf_model_file = model_path / f"{cls._metadata_filename()}.tf_model" - loaded_data = rasa.utils.io.pickle_load( - model_path / f"{cls._metadata_filename()}.data_example.pkl" + + # load data example + loaded_data = deserialize_nested_feature_arrays( + str(model_path / f"{cls._metadata_filename()}.data_example.st"), + str(model_path / f"{cls._metadata_filename()}.data_example_metadata.json"), ) - label_data = rasa.utils.io.pickle_load( - model_path / f"{cls._metadata_filename()}.label_data.pkl" + # load label data + loaded_label_data = deserialize_nested_feature_arrays( + str(model_path / f"{cls._metadata_filename()}.label_data.st"), + str(model_path / f"{cls._metadata_filename()}.label_data_metadata.json"), ) - fake_features = rasa.utils.io.pickle_load( - model_path / f"{cls._metadata_filename()}.fake_features.pkl" + label_data = RasaModelData(data=loaded_label_data) + + # load fake features + metadata = rasa.shared.utils.io.read_json_file( + model_path / f"{cls._metadata_filename()}.fake_features_metadata.json" ) - label_data = RasaModelData(data=label_data) - priority = rasa.utils.io.json_unpickle( - model_path / f"{cls._metadata_filename()}.priority.pkl" + fake_features = load_features( + str(model_path / f"{cls._metadata_filename()}.fake_features.st"), metadata + ) + + priority = rasa.shared.utils.io.read_json_file( + model_path / f"{cls._metadata_filename()}.priority.json" ) entity_tag_specs = rasa.shared.utils.io.read_json_file( model_path / f"{cls._metadata_filename()}.entity_tag_specs.json" @@ -1023,8 +1048,8 @@ def _load_model_utilities(cls, model_path: Path) -> Dict[Text, Any]: ) for tag_spec in entity_tag_specs ] - model_config = rasa.utils.io.pickle_load( - model_path / f"{cls._metadata_filename()}.meta.pkl" + model_config = rasa.shared.utils.io.read_json_file( + model_path / f"{cls._metadata_filename()}.meta.json" ) return { @@ -1070,7 +1095,7 @@ def _load( ) -> TEDPolicy: featurizer = TrackerFeaturizer.load(model_path) - if not (model_path / f"{cls._metadata_filename()}.data_example.pkl").is_file(): + if not (model_path / f"{cls._metadata_filename()}.data_example.st").is_file(): return cls( config, model_storage, diff --git a/rasa/core/policies/unexpected_intent_policy.py b/rasa/core/policies/unexpected_intent_policy.py index d5b39a561b82..ca788662f133 100644 --- a/rasa/core/policies/unexpected_intent_policy.py +++ b/rasa/core/policies/unexpected_intent_policy.py @@ -5,6 +5,7 @@ import numpy as np import tensorflow as tf + import rasa.utils.common from rasa.engine.graph import ExecutionContext from rasa.engine.recipes.default_recipe import DefaultV1Recipe @@ -16,6 +17,7 @@ from rasa.shared.core.trackers import DialogueStateTracker from rasa.shared.core.constants import SLOTS, ACTIVE_LOOP, ACTION_UNLIKELY_INTENT_NAME from rasa.shared.core.events import UserUttered, ActionExecuted +import rasa.shared.utils.io from rasa.shared.nlu.constants import ( INTENT, TEXT, @@ -103,8 +105,6 @@ ) from rasa.utils.tensorflow import layers from rasa.utils.tensorflow.model_data import RasaModelData, FeatureArray, Data - -import rasa.utils.io as io_utils from rasa.core.exceptions import RasaCoreException from rasa.shared.utils import common @@ -881,9 +881,12 @@ def persist_model_utilities(self, model_path: Path) -> None: model_path: Path where model is to be persisted """ super().persist_model_utilities(model_path) - io_utils.pickle_dump( - model_path / f"{self._metadata_filename()}.label_quantiles.pkl", - self.label_quantiles, + + from safetensors.numpy import save_file + + save_file( + {str(k): np.array(v) for k, v in self.label_quantiles.items()}, + model_path / f"{self._metadata_filename()}.label_quantiles.st", ) @classmethod @@ -894,9 +897,14 @@ def _load_model_utilities(cls, model_path: Path) -> Dict[Text, Any]: model_path: Path where model is to be persisted. """ model_utilties = super()._load_model_utilities(model_path) - label_quantiles = io_utils.pickle_load( - model_path / f"{cls._metadata_filename()}.label_quantiles.pkl" + + from safetensors.numpy import load_file + + loaded_label_quantiles = load_file( + model_path / f"{cls._metadata_filename()}.label_quantiles.st" ) + label_quantiles = {int(k): list(v) for k, v in loaded_label_quantiles.items()} + model_utilties.update({"label_quantiles": label_quantiles}) return model_utilties diff --git a/rasa/core/processor.py b/rasa/core/processor.py index ec6ef1260a3e..fc628c7a7247 100644 --- a/rasa/core/processor.py +++ b/rasa/core/processor.py @@ -66,7 +66,11 @@ ENTITIES, INTENT, INTENT_NAME_KEY, + INTENT_RESPONSE_KEY, PREDICTED_CONFIDENCE_KEY, + FULL_RETRIEVAL_INTENT_NAME_KEY, + RESPONSE_SELECTOR, + RESPONSE, TEXT, ) from rasa.utils.endpoints import EndpointConfig @@ -721,6 +725,7 @@ async def parse_message( message, tracker, only_output_properties ) + self._update_full_retrieval_intent(parse_data) structlogger.debug( "processor.message.parse", parse_data_text=copy.deepcopy(parse_data["text"]), @@ -732,6 +737,23 @@ async def parse_message( return parse_data + def _update_full_retrieval_intent(self, parse_data: Dict[Text, Any]) -> None: + """Update the parse data with the full retrieval intent. + + Args: + parse_data: Message parse data to update. + """ + intent_name = parse_data.get(INTENT, {}).get(INTENT_NAME_KEY) + response_selector = parse_data.get(RESPONSE_SELECTOR, {}) + all_retrieval_intents = response_selector.get("all_retrieval_intents", []) + if intent_name and intent_name in all_retrieval_intents: + retrieval_intent = ( + response_selector.get(intent_name, {}) + .get(RESPONSE, {}) + .get(INTENT_RESPONSE_KEY) + ) + parse_data[INTENT][FULL_RETRIEVAL_INTENT_NAME_KEY] = retrieval_intent + def _parse_message_with_graph( self, message: UserMessage, diff --git a/rasa/core/run.py b/rasa/core/run.py index 5270162809dd..3a8133613c3f 100644 --- a/rasa/core/run.py +++ b/rasa/core/run.py @@ -1,9 +1,19 @@ import asyncio import logging import uuid +import platform import os from functools import partial -from typing import Any, List, Optional, TYPE_CHECKING, Text, Union, Dict +from typing import ( + Any, + Callable, + List, + Optional, + Text, + Tuple, + Union, + Dict, +) import rasa.core.utils from rasa.plugin import plugin_manager @@ -23,8 +33,6 @@ from sanic import Sanic from asyncio import AbstractEventLoop -if TYPE_CHECKING: - from aiohttp import ClientSession logger = logging.getLogger() # get the root logger @@ -80,6 +88,14 @@ def _create_app_without_api(cors: Optional[Union[Text, List[Text]]] = None) -> S return app +def _is_apple_silicon_system() -> bool: + # check if the system is MacOS + if platform.system().lower() != "darwin": + return False + # check for arm architecture, indicating apple silicon + return platform.machine().startswith("arm") or os.uname().machine.startswith("arm") + + def configure_app( input_channels: Optional[List["InputChannel"]] = None, cors: Optional[Union[Text, List[Text], None]] = None, @@ -99,6 +115,9 @@ def configure_app( syslog_port: Optional[int] = None, syslog_protocol: Optional[Text] = None, request_timeout: Optional[int] = None, + server_listeners: Optional[List[Tuple[Callable, Text]]] = None, + use_uvloop: Optional[bool] = True, + keep_alive_timeout: int = constants.DEFAULT_KEEP_ALIVE_TIMEOUT, ) -> Sanic: """Run the agent.""" rasa.core.utils.configure_file_logging( @@ -118,6 +137,14 @@ def configure_app( else: app = _create_app_without_api(cors) + app.config.KEEP_ALIVE_TIMEOUT = keep_alive_timeout + if _is_apple_silicon_system() or not use_uvloop: + app.config.USE_UVLOOP = False + # some library still sets the loop to uvloop, even if disabled for sanic + # using uvloop leads to breakingio errors, see + # https://rasahq.atlassian.net/browse/ENG-667 + asyncio.set_event_loop_policy(None) + if input_channels: channels.channel.register(input_channels, app, route=route) else: @@ -150,6 +177,10 @@ async def run_cmdline_io(running_app: Sanic) -> None: app.add_task(run_cmdline_io) + if server_listeners: + for (listener, event) in server_listeners: + app.register_listener(listener, event) + return app @@ -179,6 +210,7 @@ def serve_application( syslog_port: Optional[int] = None, syslog_protocol: Optional[Text] = None, request_timeout: Optional[int] = None, + server_listeners: Optional[List[Tuple[Callable, Text]]] = None, ) -> None: """Run the API entrypoint.""" if not channel and not credentials: @@ -204,6 +236,7 @@ def serve_application( syslog_port=syslog_port, syslog_protocol=syslog_protocol, request_timeout=request_timeout, + server_listeners=server_listeners, ) ssl_context = server.create_ssl_context( @@ -217,7 +250,7 @@ def serve_application( partial(load_agent_on_start, model_path, endpoints, remote_storage), "before_server_start", ) - app.register_listener(create_connection_pools, "after_server_start") + app.register_listener(close_resources, "after_server_stop") number_of_workers = rasa.core.utils.number_of_sanic_workers( @@ -279,44 +312,3 @@ async def close_resources(app: Sanic, _: AbstractEventLoop) -> None: event_broker = current_agent.tracker_store.event_broker if event_broker: await event_broker.close() - - action_endpoint = current_agent.action_endpoint - if action_endpoint: - await action_endpoint.session.close() - - model_server = current_agent.model_server - if model_server: - await model_server.session.close() - - -async def create_connection_pools(app: Sanic, _: AbstractEventLoop) -> None: - """Create connection pools for the agent's action server and model server.""" - current_agent = getattr(app.ctx, "agent", None) - if not current_agent: - logger.debug("No agent found after server start.") - return None - - create_action_endpoint_connection_pool(current_agent) - create_model_server_connection_pool(current_agent) - - return None - - -def create_action_endpoint_connection_pool(agent: Agent) -> Optional["ClientSession"]: - """Create a connection pool for the action endpoint.""" - action_endpoint = agent.action_endpoint - if not action_endpoint: - logger.debug("No action endpoint found after server start.") - return None - - return action_endpoint.session - - -def create_model_server_connection_pool(agent: Agent) -> Optional["ClientSession"]: - """Create a connection pool for the model server.""" - model_server = agent.model_server - if not model_server: - logger.debug("No model server endpoint found after server start.") - return None - - return model_server.session diff --git a/rasa/core/tracker_store.py b/rasa/core/tracker_store.py index 93632cd3d779..a91f9f5c6b87 100644 --- a/rasa/core/tracker_store.py +++ b/rasa/core/tracker_store.py @@ -1291,6 +1291,9 @@ def _event_query( self, session: "Session", sender_id: Text, fetch_events_from_all_sessions: bool ) -> "Query": """Provide the query to retrieve the conversation events for a specific sender. + The events are ordered by ID to ensure correct sequence of events. + As `timestamp` is not guaranteed to be unique and low-precision (float), it + cannot be used to order the events. Args: session: Current database session. @@ -1325,7 +1328,7 @@ def _event_query( ) ) - return event_query.order_by(self.SQLEvent.timestamp) + return event_query.order_by(self.SQLEvent.id) async def save(self, tracker: DialogueStateTracker) -> None: """Update database with events from the current conversation.""" diff --git a/rasa/nlu/classifiers/diet_classifier.py b/rasa/nlu/classifiers/diet_classifier.py index 1cc65c89b3c9..b53eb5db8d76 100644 --- a/rasa/nlu/classifiers/diet_classifier.py +++ b/rasa/nlu/classifiers/diet_classifier.py @@ -1,18 +1,17 @@ from __future__ import annotations + import copy import logging from collections import defaultdict from pathlib import Path - -from rasa.exceptions import ModelNotFound -from rasa.nlu.featurizers.featurizer import Featurizer +from typing import Any, Dict, List, Optional, Text, Tuple, Union, TypeVar, Type import numpy as np import scipy.sparse import tensorflow as tf -from typing import Any, Dict, List, Optional, Text, Tuple, Union, TypeVar, Type - +from rasa.exceptions import ModelNotFound +from rasa.nlu.featurizers.featurizer import Featurizer from rasa.engine.graph import ExecutionContext, GraphComponent from rasa.engine.recipes.default_recipe import DefaultV1Recipe from rasa.engine.storage.resource import Resource @@ -20,18 +19,21 @@ from rasa.nlu.extractors.extractor import EntityExtractorMixin from rasa.nlu.classifiers.classifier import IntentClassifier import rasa.shared.utils.io -import rasa.utils.io as io_utils import rasa.nlu.utils.bilou_utils as bilou_utils from rasa.shared.constants import DIAGNOSTIC_DATA from rasa.nlu.extractors.extractor import EntityTagSpec from rasa.nlu.classifiers import LABEL_RANKING_LENGTH from rasa.utils import train_utils from rasa.utils.tensorflow import rasa_layers +from rasa.utils.tensorflow.feature_array import ( + FeatureArray, + serialize_nested_feature_arrays, + deserialize_nested_feature_arrays, +) from rasa.utils.tensorflow.models import RasaModel, TransformerRasaModel from rasa.utils.tensorflow.model_data import ( RasaModelData, FeatureSignature, - FeatureArray, ) from rasa.nlu.constants import TOKENS_NAMES, DEFAULT_TRANSFORMER_SIZE from rasa.shared.nlu.constants import ( @@ -50,6 +52,7 @@ from rasa.shared.nlu.training_data.training_data import TrainingData from rasa.shared.nlu.training_data.message import Message from rasa.utils.tensorflow.constants import ( + DROP_SMALL_LAST_BATCH, LABEL, IDS, HIDDEN_LAYERS_SIZES, @@ -117,7 +120,6 @@ POSSIBLE_TAGS = [ENTITY_ATTRIBUTE_TYPE, ENTITY_ATTRIBUTE_ROLE, ENTITY_ATTRIBUTE_GROUP] - DIETClassifierT = TypeVar("DIETClassifierT", bound="DIETClassifier") @@ -288,6 +290,9 @@ def get_default_config() -> Dict[Text, Any]: # a few steps, as the compilation of the graph tends to take more time than # running it. It is recommended to not adjust the optimization parameter. RUN_EAGERLY: False, + # Determines whether the last batch should be dropped if it contains fewer + # than half a batch size of examples + DROP_SMALL_LAST_BATCH: False, } def __init__( @@ -931,6 +936,7 @@ def train(self, training_data: TrainingData) -> Resource: self.component_config[BATCH_STRATEGY], self.component_config[EVAL_NUM_EXAMPLES], self.component_config[RANDOM_SEED], + drop_small_last_batch=self.component_config[DROP_SMALL_LAST_BATCH], ) callbacks = train_utils.create_common_callbacks( self.component_config[EPOCHS], @@ -1080,18 +1086,24 @@ def persist(self) -> None: self.model.save(str(tf_model_file)) - io_utils.pickle_dump( - model_path / f"{file_name}.data_example.pkl", self._data_example - ) - io_utils.pickle_dump( - model_path / f"{file_name}.sparse_feature_sizes.pkl", - self._sparse_feature_sizes, + # save data example + serialize_nested_feature_arrays( + self._data_example, + model_path / f"{file_name}.data_example.st", + model_path / f"{file_name}.data_example_metadata.json", ) - io_utils.pickle_dump( - model_path / f"{file_name}.label_data.pkl", + # save label data + serialize_nested_feature_arrays( dict(self._label_data.data) if self._label_data is not None else {}, + model_path / f"{file_name}.label_data.st", + model_path / f"{file_name}.label_data_metadata.json", ) - io_utils.json_pickle( + + rasa.shared.utils.io.dump_obj_as_json_to_file( + model_path / f"{file_name}.sparse_feature_sizes.json", + self._sparse_feature_sizes, + ) + rasa.shared.utils.io.dump_obj_as_json_to_file( model_path / f"{file_name}.index_label_id_mapping.json", self.index_label_id_mapping, ) @@ -1180,15 +1192,22 @@ def _load_from_files( ]: file_name = cls.__name__ - data_example = io_utils.pickle_load( - model_path / f"{file_name}.data_example.pkl" + # load data example + data_example = deserialize_nested_feature_arrays( + str(model_path / f"{file_name}.data_example.st"), + str(model_path / f"{file_name}.data_example_metadata.json"), ) - label_data = io_utils.pickle_load(model_path / f"{file_name}.label_data.pkl") - label_data = RasaModelData(data=label_data) - sparse_feature_sizes = io_utils.pickle_load( - model_path / f"{file_name}.sparse_feature_sizes.pkl" + # load label data + loaded_label_data = deserialize_nested_feature_arrays( + str(model_path / f"{file_name}.label_data.st"), + str(model_path / f"{file_name}.label_data_metadata.json"), + ) + label_data = RasaModelData(data=loaded_label_data) + + sparse_feature_sizes = rasa.shared.utils.io.read_json_file( + model_path / f"{file_name}.sparse_feature_sizes.json" ) - index_label_id_mapping = io_utils.json_unpickle( + index_label_id_mapping = rasa.shared.utils.io.read_json_file( model_path / f"{file_name}.index_label_id_mapping.json" ) entity_tag_specs = rasa.shared.utils.io.read_json_file( @@ -1208,7 +1227,6 @@ def _load_from_files( for tag_spec in entity_tag_specs ] - # jsonpickle converts dictionary keys to strings index_label_id_mapping = { int(key): value for key, value in index_label_id_mapping.items() } diff --git a/rasa/nlu/classifiers/logistic_regression_classifier.py b/rasa/nlu/classifiers/logistic_regression_classifier.py index c652d20af9c0..46303a11697e 100644 --- a/rasa/nlu/classifiers/logistic_regression_classifier.py +++ b/rasa/nlu/classifiers/logistic_regression_classifier.py @@ -1,20 +1,19 @@ import logging from typing import Any, Text, Dict, List, Type, Tuple -import joblib from scipy.sparse import hstack, vstack, csr_matrix from sklearn.linear_model import LogisticRegression +from rasa.engine.graph import ExecutionContext, GraphComponent +from rasa.engine.recipes.default_recipe import DefaultV1Recipe from rasa.engine.storage.resource import Resource from rasa.engine.storage.storage import ModelStorage -from rasa.engine.recipes.default_recipe import DefaultV1Recipe -from rasa.engine.graph import ExecutionContext, GraphComponent from rasa.nlu.classifiers import LABEL_RANKING_LENGTH -from rasa.nlu.featurizers.featurizer import Featurizer from rasa.nlu.classifiers.classifier import IntentClassifier -from rasa.shared.nlu.training_data.training_data import TrainingData -from rasa.shared.nlu.training_data.message import Message +from rasa.nlu.featurizers.featurizer import Featurizer from rasa.shared.nlu.constants import TEXT, INTENT +from rasa.shared.nlu.training_data.message import Message +from rasa.shared.nlu.training_data.training_data import TrainingData from rasa.utils.tensorflow.constants import RANKING_LENGTH logger = logging.getLogger(__name__) @@ -158,9 +157,11 @@ def process(self, messages: List[Message]) -> List[Message]: def persist(self) -> None: """Persist this model into the passed directory.""" + import skops.io as sio + with self._model_storage.write_to(self._resource) as model_dir: - path = model_dir / f"{self._resource.name}.joblib" - joblib.dump(self.clf, path) + path = model_dir / f"{self._resource.name}.skops" + sio.dump(self.clf, path) logger.debug(f"Saved intent classifier to '{path}'.") @classmethod @@ -173,9 +174,21 @@ def load( **kwargs: Any, ) -> "LogisticRegressionClassifier": """Loads trained component (see parent class for full docstring).""" + import skops.io as sio + try: with model_storage.read_from(resource) as model_dir: - classifier = joblib.load(model_dir / f"{resource.name}.joblib") + classifier_file = model_dir / f"{resource.name}.skops" + unknown_types = sio.get_untrusted_types(file=classifier_file) + + if unknown_types: + logger.debug( + f"Untrusted types ({unknown_types}) found when " + f"loading {classifier_file}!", + ) + raise ValueError() + + classifier = sio.load(classifier_file, trusted=unknown_types) component = cls( config, execution_context.node_name, model_storage, resource ) diff --git a/rasa/nlu/classifiers/sklearn_intent_classifier.py b/rasa/nlu/classifiers/sklearn_intent_classifier.py index 5c941d3d8806..3aa656f0f3ba 100644 --- a/rasa/nlu/classifiers/sklearn_intent_classifier.py +++ b/rasa/nlu/classifiers/sklearn_intent_classifier.py @@ -1,6 +1,6 @@ from __future__ import annotations + import logging -from rasa.nlu.featurizers.dense_featurizer.dense_featurizer import DenseFeaturizer import typing import warnings from typing import Any, Dict, List, Optional, Text, Tuple, Type @@ -8,18 +8,18 @@ import numpy as np import rasa.shared.utils.io -import rasa.utils.io as io_utils from rasa.engine.graph import GraphComponent, ExecutionContext from rasa.engine.recipes.default_recipe import DefaultV1Recipe from rasa.engine.storage.resource import Resource from rasa.engine.storage.storage import ModelStorage -from rasa.shared.constants import DOCS_URL_TRAINING_DATA_NLU from rasa.nlu.classifiers import LABEL_RANKING_LENGTH +from rasa.nlu.classifiers.classifier import IntentClassifier +from rasa.nlu.featurizers.dense_featurizer.dense_featurizer import DenseFeaturizer +from rasa.shared.constants import DOCS_URL_TRAINING_DATA_NLU from rasa.shared.exceptions import RasaException from rasa.shared.nlu.constants import TEXT -from rasa.nlu.classifiers.classifier import IntentClassifier -from rasa.shared.nlu.training_data.training_data import TrainingData from rasa.shared.nlu.training_data.message import Message +from rasa.shared.nlu.training_data.training_data import TrainingData from rasa.utils.tensorflow.constants import FEATURIZERS logger = logging.getLogger(__name__) @@ -266,14 +266,20 @@ def predict(self, X: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: def persist(self) -> None: """Persist this model into the passed directory.""" + import skops.io as sio + with self._model_storage.write_to(self._resource) as model_dir: file_name = self.__class__.__name__ - classifier_file_name = model_dir / f"{file_name}_classifier.pkl" - encoder_file_name = model_dir / f"{file_name}_encoder.pkl" + classifier_file_name = model_dir / f"{file_name}_classifier.skops" + encoder_file_name = model_dir / f"{file_name}_encoder.json" if self.clf and self.le: - io_utils.json_pickle(encoder_file_name, self.le.classes_) - io_utils.json_pickle(classifier_file_name, self.clf.best_estimator_) + # convert self.le.classes_ (numpy array of strings) to a list in order + # to use json dump + rasa.shared.utils.io.dump_obj_as_json_to_file( + encoder_file_name, list(self.le.classes_) + ) + sio.dump(self.clf.best_estimator_, classifier_file_name) @classmethod def load( @@ -286,21 +292,36 @@ def load( ) -> SklearnIntentClassifier: """Loads trained component (see parent class for full docstring).""" from sklearn.preprocessing import LabelEncoder + import skops.io as sio try: with model_storage.read_from(resource) as model_dir: file_name = cls.__name__ - classifier_file = model_dir / f"{file_name}_classifier.pkl" + classifier_file = model_dir / f"{file_name}_classifier.skops" if classifier_file.exists(): - classifier = io_utils.json_unpickle(classifier_file) + unknown_types = sio.get_untrusted_types(file=classifier_file) - encoder_file = model_dir / f"{file_name}_encoder.pkl" - classes = io_utils.json_unpickle(encoder_file) - encoder = LabelEncoder() - encoder.classes_ = classes + if unknown_types: + logger.error( + f"Untrusted types ({unknown_types}) found when " + f"loading {classifier_file}!" + ) + raise ValueError() + else: + classifier = sio.load(classifier_file, trusted=unknown_types) + + encoder_file = model_dir / f"{file_name}_encoder.json" + classes = rasa.shared.utils.io.read_json_file(encoder_file) - return cls(config, model_storage, resource, classifier, encoder) + encoder = LabelEncoder() + intent_classifier = cls( + config, model_storage, resource, classifier, encoder + ) + # convert list of strings (class labels) back to numpy array of + # strings + intent_classifier.transform_labels_str2num(classes) + return intent_classifier except ValueError: logger.debug( f"Failed to load '{cls.__name__}' from model storage. Resource " diff --git a/rasa/nlu/extractors/crf_entity_extractor.py b/rasa/nlu/extractors/crf_entity_extractor.py index 1332c250d55a..357d6044fec8 100644 --- a/rasa/nlu/extractors/crf_entity_extractor.py +++ b/rasa/nlu/extractors/crf_entity_extractor.py @@ -1,12 +1,12 @@ from __future__ import annotations -from collections import OrderedDict -from enum import Enum import logging import typing +from collections import OrderedDict +from enum import Enum +from typing import Any, Dict, List, Optional, Text, Tuple, Callable, Type import numpy as np -from typing import Any, Dict, List, Optional, Text, Tuple, Callable, Type import rasa.nlu.utils.bilou_utils as bilou_utils import rasa.shared.utils.io @@ -15,13 +15,12 @@ from rasa.engine.recipes.default_recipe import DefaultV1Recipe from rasa.engine.storage.resource import Resource from rasa.engine.storage.storage import ModelStorage +from rasa.nlu.constants import TOKENS_NAMES +from rasa.nlu.extractors.extractor import EntityExtractorMixin from rasa.nlu.test import determine_token_labels from rasa.nlu.tokenizers.spacy_tokenizer import POS_TAG_KEY -from rasa.nlu.extractors.extractor import EntityExtractorMixin from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer -from rasa.shared.nlu.training_data.training_data import TrainingData -from rasa.shared.nlu.training_data.message import Message -from rasa.nlu.constants import TOKENS_NAMES +from rasa.shared.constants import DOCS_URL_COMPONENTS from rasa.shared.nlu.constants import ( TEXT, ENTITIES, @@ -32,7 +31,8 @@ SPLIT_ENTITIES_BY_COMMA, SPLIT_ENTITIES_BY_COMMA_DEFAULT_VALUE, ) -from rasa.shared.constants import DOCS_URL_COMPONENTS +from rasa.shared.nlu.training_data.message import Message +from rasa.shared.nlu.training_data.training_data import TrainingData from rasa.utils.tensorflow.constants import BILOU_FLAG, FEATURIZERS logger = logging.getLogger(__name__) @@ -41,6 +41,9 @@ from sklearn_crfsuite import CRF +CONFIG_FEATURES = "features" + + class CRFToken: def __init__( self, @@ -60,6 +63,29 @@ def __init__( self.entity_role_tag = entity_role_tag self.entity_group_tag = entity_group_tag + def to_dict(self) -> Dict[str, Any]: + return { + "text": self.text, + "pos_tag": self.pos_tag, + "pattern": self.pattern, + "dense_features": [str(x) for x in list(self.dense_features)], + "entity_tag": self.entity_tag, + "entity_role_tag": self.entity_role_tag, + "entity_group_tag": self.entity_group_tag, + } + + @classmethod + def create_from_dict(cls, data: Dict[str, Any]) -> "CRFToken": + return cls( + data["text"], + data["pos_tag"], + data["pattern"], + np.array([float(x) for x in data["dense_features"]]), + data["entity_tag"], + data["entity_role_tag"], + data["entity_group_tag"], + ) + class CRFEntityExtractorOptions(str, Enum): """Features that can be used for the 'CRFEntityExtractor'.""" @@ -137,7 +163,7 @@ def get_default_config() -> Dict[Text, Any]: # "is the preceding token in title case?" # POS features require SpacyTokenizer # pattern feature require RegexFeaturizer - CRFEntityExtractor.CONFIG_FEATURES: [ + CONFIG_FEATURES: [ [ CRFEntityExtractorOptions.LOW, CRFEntityExtractorOptions.TITLE, @@ -200,7 +226,7 @@ def __init__( ) def _validate_configuration(self) -> None: - if len(self.component_config.get(self.CONFIG_FEATURES, [])) % 2 != 1: + if len(self.component_config.get(CONFIG_FEATURES, [])) % 2 != 1: raise ValueError( "Need an odd number of crf feature lists to have a center word." ) @@ -251,9 +277,11 @@ def train(self, training_data: TrainingData) -> Resource: ] dataset = [self._convert_to_crf_tokens(example) for example in entity_examples] - self._train_model(dataset) + self.entity_taggers = self.train_model( + dataset, self.component_config, self.crf_order + ) - self.persist() + self.persist(dataset) return self._resource @@ -299,7 +327,9 @@ def extract_entities(self, message: Message) -> List[Dict[Text, Any]]: if include_tag_features: self._add_tag_to_crf_token(crf_tokens, predictions) - features = self._crf_tokens_to_features(crf_tokens, include_tag_features) + features = self._crf_tokens_to_features( + crf_tokens, self.component_config, include_tag_features + ) predictions[tag_name] = entity_tagger.predict_marginals_single(features) # convert predictions into a list of tags and a list of confidences @@ -389,27 +419,25 @@ def load( **kwargs: Any, ) -> CRFEntityExtractor: """Loads trained component (see parent class for full docstring).""" - import joblib - try: - entity_taggers = OrderedDict() with model_storage.read_from(resource) as model_dir: - # We have to load in the same order as we persisted things as otherwise - # the predictions might be off - file_names = sorted(model_dir.glob("**/*.pkl")) - if not file_names: - logger.debug( - "Failed to load model for 'CRFEntityExtractor'. " - "Maybe you did not provide enough training data and " - "no model was trained." - ) - return cls(config, model_storage, resource) + dataset = rasa.shared.utils.io.read_json_file( + model_dir / "crf_dataset.json" + ) + crf_order = rasa.shared.utils.io.read_json_file( + model_dir / "crf_order.json" + ) + + dataset = [ + [CRFToken.create_from_dict(token_data) for token_data in sub_list] + for sub_list in dataset + ] - for file_name in file_names: - name = file_name.stem[1:] - entity_taggers[name] = joblib.load(file_name) + entity_taggers = cls.train_model(dataset, config, crf_order) - return cls(config, model_storage, resource, entity_taggers) + entity_extractor = cls(config, model_storage, resource, entity_taggers) + entity_extractor.crf_order = crf_order + return entity_extractor except ValueError: logger.warning( f"Failed to load {cls.__name__} from model storage. Resource " @@ -417,23 +445,29 @@ def load( ) return cls(config, model_storage, resource) - def persist(self) -> None: + def persist(self, dataset: List[List[CRFToken]]) -> None: """Persist this model into the passed directory.""" - import joblib - with self._model_storage.write_to(self._resource) as model_dir: - if self.entity_taggers: - for idx, (name, entity_tagger) in enumerate( - self.entity_taggers.items() - ): - model_file_name = model_dir / f"{idx}{name}.pkl" - joblib.dump(entity_tagger, model_file_name) + data_to_store = [ + [token.to_dict() for token in sub_list] for sub_list in dataset + ] + rasa.shared.utils.io.dump_obj_as_json_to_file( + model_dir / "crf_dataset.json", data_to_store + ) + rasa.shared.utils.io.dump_obj_as_json_to_file( + model_dir / "crf_order.json", self.crf_order + ) + + @classmethod def _crf_tokens_to_features( - self, crf_tokens: List[CRFToken], include_tag_features: bool = False + cls, + crf_tokens: List[CRFToken], + config: Dict[str, Any], + include_tag_features: bool = False, ) -> List[Dict[Text, Any]]: """Convert the list of tokens into discrete features.""" - configured_features = self.component_config[self.CONFIG_FEATURES] + configured_features = config[CONFIG_FEATURES] sentence_features = [] for token_idx in range(len(crf_tokens)): @@ -444,28 +478,31 @@ def _crf_tokens_to_features( half_window_size = window_size // 2 window_range = range(-half_window_size, half_window_size + 1) - token_features = self._create_features_for_token( + token_features = cls._create_features_for_token( crf_tokens, token_idx, half_window_size, window_range, include_tag_features, + config, ) sentence_features.append(token_features) return sentence_features + @classmethod def _create_features_for_token( - self, + cls, crf_tokens: List[CRFToken], token_idx: int, half_window_size: int, window_range: range, include_tag_features: bool, + config: Dict[str, Any], ) -> Dict[Text, Any]: """Convert a token into discrete features including words before and after.""" - configured_features = self.component_config[self.CONFIG_FEATURES] + configured_features = config[CONFIG_FEATURES] prefixes = [str(i) for i in window_range] token_features = {} @@ -505,13 +542,13 @@ def _create_features_for_token( # set in the training data, 'matched' is either 'True' or # 'False' depending on whether the token actually matches the # pattern or not - regex_patterns = self.function_dict[feature](token) + regex_patterns = cls.function_dict[feature](token) for pattern_name, matched in regex_patterns.items(): token_features[ f"{prefix}:{feature}:{pattern_name}" ] = matched else: - value = self.function_dict[feature](token) + value = cls.function_dict[feature](token) token_features[f"{prefix}:{feature}"] = value return token_features @@ -635,38 +672,46 @@ def _get_tags(self, message: Message) -> Dict[Text, List[Text]]: return tags - def _train_model(self, df_train: List[List[CRFToken]]) -> None: + @classmethod + def train_model( + cls, + df_train: List[List[CRFToken]], + config: Dict[str, Any], + crf_order: List[str], + ) -> OrderedDict[str, CRF]: """Train the crf tagger based on the training data.""" import sklearn_crfsuite - self.entity_taggers = OrderedDict() + entity_taggers = OrderedDict() - for tag_name in self.crf_order: + for tag_name in crf_order: logger.debug(f"Training CRF for '{tag_name}'.") # add entity tag features for second level CRFs include_tag_features = tag_name != ENTITY_ATTRIBUTE_TYPE X_train = ( - self._crf_tokens_to_features(sentence, include_tag_features) + cls._crf_tokens_to_features(sentence, config, include_tag_features) for sentence in df_train ) y_train = ( - self._crf_tokens_to_tags(sentence, tag_name) for sentence in df_train + cls._crf_tokens_to_tags(sentence, tag_name) for sentence in df_train ) entity_tagger = sklearn_crfsuite.CRF( algorithm="lbfgs", # coefficient for L1 penalty - c1=self.component_config["L1_c"], + c1=config["L1_c"], # coefficient for L2 penalty - c2=self.component_config["L2_c"], + c2=config["L2_c"], # stop earlier - max_iterations=self.component_config["max_iterations"], + max_iterations=config["max_iterations"], # include transitions that are possible, but not observed all_possible_transitions=True, ) entity_tagger.fit(X_train, y_train) - self.entity_taggers[tag_name] = entity_tagger + entity_taggers[tag_name] = entity_tagger logger.debug("Training finished.") + + return entity_taggers diff --git a/rasa/nlu/featurizers/sparse_featurizer/count_vectors_featurizer.py b/rasa/nlu/featurizers/sparse_featurizer/count_vectors_featurizer.py index 0c76b71fa6ea..98cecba9ca3e 100644 --- a/rasa/nlu/featurizers/sparse_featurizer/count_vectors_featurizer.py +++ b/rasa/nlu/featurizers/sparse_featurizer/count_vectors_featurizer.py @@ -1,29 +1,31 @@ from __future__ import annotations + import logging import re +from typing import Any, Dict, List, Optional, Text, Tuple, Set, Type, Union + +import numpy as np import scipy.sparse -from typing import Any, Dict, List, Optional, Text, Tuple, Set, Type -from rasa.nlu.tokenizers.tokenizer import Tokenizer +from sklearn.feature_extraction.text import CountVectorizer import rasa.shared.utils.io from rasa.engine.graph import GraphComponent, ExecutionContext from rasa.engine.recipes.default_recipe import DefaultV1Recipe from rasa.engine.storage.resource import Resource from rasa.engine.storage.storage import ModelStorage -from rasa.nlu.featurizers.sparse_featurizer.sparse_featurizer import SparseFeaturizer -from rasa.nlu.utils.spacy_utils import SpacyModel -from rasa.shared.constants import DOCS_URL_COMPONENTS -import rasa.utils.io as io_utils -from sklearn.feature_extraction.text import CountVectorizer -from rasa.shared.nlu.training_data.training_data import TrainingData -from rasa.shared.nlu.training_data.message import Message -from rasa.shared.exceptions import RasaException, FileIOException from rasa.nlu.constants import ( TOKENS_NAMES, MESSAGE_ATTRIBUTES, DENSE_FEATURIZABLE_ATTRIBUTES, ) +from rasa.nlu.featurizers.sparse_featurizer.sparse_featurizer import SparseFeaturizer +from rasa.nlu.tokenizers.tokenizer import Tokenizer +from rasa.nlu.utils.spacy_utils import SpacyModel +from rasa.shared.constants import DOCS_URL_COMPONENTS +from rasa.shared.exceptions import RasaException, FileIOException from rasa.shared.nlu.constants import TEXT, INTENT, INTENT_RESPONSE_KEY, ACTION_NAME +from rasa.shared.nlu.training_data.message import Message +from rasa.shared.nlu.training_data.training_data import TrainingData BUFFER_SLOTS_PREFIX = "buf_" @@ -686,6 +688,31 @@ def _is_any_model_trained( """Check if any model got trained.""" return any(value is not None for value in attribute_vocabularies.values()) + @staticmethod + def convert_vocab( + vocab: Dict[str, Union[int, Optional[Dict[str, int]]]], to_int: bool + ) -> Dict[str, Union[None, int, np.int64, Dict[str, Union[int, np.int64]]]]: + """Converts numpy integers in the vocabulary to Python integers.""" + + def convert_value(value: int) -> Union[int, np.int64]: + """Helper function to convert a single value based on to_int flag.""" + return int(value) if to_int else np.int64(value) + + result_dict: Dict[ + str, Union[None, int, np.int64, Dict[str, Union[int, np.int64]]] + ] = {} + for key, sub_dict in vocab.items(): + if isinstance(sub_dict, int): + result_dict[key] = convert_value(sub_dict) + elif not sub_dict: + result_dict[key] = None + else: + result_dict[key] = { + sub_key: convert_value(value) for sub_key, value in sub_dict.items() + } + + return result_dict + def persist(self) -> None: """Persist this model into the passed directory. @@ -699,17 +726,18 @@ def persist(self) -> None: attribute_vocabularies = self._collect_vectorizer_vocabularies() if self._is_any_model_trained(attribute_vocabularies): # Definitely need to persist some vocabularies - featurizer_file = model_dir / "vocabularies.pkl" + featurizer_file = model_dir / "vocabularies.json" # Only persist vocabulary from one attribute if `use_shared_vocab`. # Can be loaded and distributed to all attributes. - vocab = ( + loaded_vocab = ( attribute_vocabularies[TEXT] if self.use_shared_vocab else attribute_vocabularies ) + vocab = self.convert_vocab(loaded_vocab, to_int=True) - io_utils.json_pickle(featurizer_file, vocab) + rasa.shared.utils.io.dump_obj_as_json_to_file(featurizer_file, vocab) # Dump OOV words separately as they might have been modified during # training @@ -784,8 +812,9 @@ def load( """Loads trained component (see parent class for full docstring).""" try: with model_storage.read_from(resource) as model_dir: - featurizer_file = model_dir / "vocabularies.pkl" - vocabulary = io_utils.json_unpickle(featurizer_file) + featurizer_file = model_dir / "vocabularies.json" + vocabulary = rasa.shared.utils.io.read_json_file(featurizer_file) + vocabulary = cls.convert_vocab(vocabulary, to_int=False) share_vocabulary = config["use_shared_vocab"] diff --git a/rasa/nlu/featurizers/sparse_featurizer/lexical_syntactic_featurizer.py b/rasa/nlu/featurizers/sparse_featurizer/lexical_syntactic_featurizer.py index 92312197755a..2c4ee3928348 100644 --- a/rasa/nlu/featurizers/sparse_featurizer/lexical_syntactic_featurizer.py +++ b/rasa/nlu/featurizers/sparse_featurizer/lexical_syntactic_featurizer.py @@ -1,9 +1,7 @@ from __future__ import annotations + import logging from collections import OrderedDict - -import scipy.sparse -import numpy as np from typing import ( Any, Dict, @@ -17,30 +15,34 @@ Union, ) +import numpy as np +import scipy.sparse + +import rasa.shared.utils.io +import rasa.utils.io from rasa.engine.graph import ExecutionContext, GraphComponent from rasa.engine.recipes.default_recipe import DefaultV1Recipe from rasa.engine.storage.resource import Resource from rasa.engine.storage.storage import ModelStorage +from rasa.nlu.constants import TOKENS_NAMES +from rasa.nlu.featurizers.sparse_featurizer.sparse_featurizer import SparseFeaturizer from rasa.nlu.tokenizers.spacy_tokenizer import POS_TAG_KEY, SpacyTokenizer from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer -from rasa.nlu.featurizers.sparse_featurizer.sparse_featurizer import SparseFeaturizer -from rasa.nlu.constants import TOKENS_NAMES from rasa.shared.constants import DOCS_URL_COMPONENTS -from rasa.shared.nlu.training_data.training_data import TrainingData -from rasa.shared.nlu.training_data.message import Message -from rasa.shared.nlu.constants import TEXT from rasa.shared.exceptions import InvalidConfigException -import rasa.shared.utils.io -import rasa.utils.io +from rasa.shared.nlu.constants import TEXT +from rasa.shared.nlu.training_data.message import Message +from rasa.shared.nlu.training_data.training_data import TrainingData logger = logging.getLogger(__name__) - END_OF_SENTENCE = "EOS" BEGIN_OF_SENTENCE = "BOS" FEATURES = "features" +SEPERATOR = "###" + @DefaultV1Recipe.register( DefaultV1Recipe.ComponentType.MESSAGE_FEATURIZER, is_trainable=True @@ -72,7 +74,7 @@ class LexicalSyntacticFeaturizer(SparseFeaturizer, GraphComponent): of the token at position `t+1`. """ - FILENAME_FEATURE_TO_IDX_DICT = "feature_to_idx_dict.pkl" + FILENAME_FEATURE_TO_IDX_DICT = "feature_to_idx_dict.json" # NOTE: "suffix5" of the token "is" will be "is". Hence, when combining multiple # prefixes, short words will be represented/encoded repeatedly. @@ -489,6 +491,32 @@ def create( """Creates a new untrained component (see parent class for full docstring).""" return cls(config, model_storage, resource, execution_context) + @staticmethod + def _restructure_feature_to_idx_dict( + loaded_data: Dict[str, Dict[str, int]], + ) -> Dict[Tuple[int, str], Dict[str, int]]: + """Reconstructs the feature to idx dict. + + When storing the feature_to_idx_dict to disk, we need to convert the tuple (key) + into a string to be able to store it via json. When loading the data + we need to reconstruct the tuple from the stored string. + + Args: + loaded_data: The loaded feature to idx dict from file. + + Returns: + The reconstructed feature_to_idx_dict + """ + feature_to_idx_dict = {} + for tuple_string, feature_value in loaded_data.items(): + # Example of tuple_string: "1###low" + index, feature_name = tuple_string.split(SEPERATOR) + + feature_key = (int(index), feature_name) + feature_to_idx_dict[feature_key] = feature_value + + return feature_to_idx_dict + @classmethod def load( cls, @@ -501,10 +529,13 @@ def load( """Loads trained component (see parent class for full docstring).""" try: with model_storage.read_from(resource) as model_path: - feature_to_idx_dict = rasa.utils.io.json_unpickle( + loaded_data = rasa.shared.utils.io.read_json_file( model_path / cls.FILENAME_FEATURE_TO_IDX_DICT, - encode_non_string_keys=True, ) + + # convert the key back into tuple + feature_to_idx_dict = cls._restructure_feature_to_idx_dict(loaded_data) + return cls( config=config, model_storage=model_storage, @@ -529,9 +560,13 @@ def persist(self) -> None: if not self._feature_to_idx_dict: return None + # as we cannot dump tuples, convert the tuple into a string + restructured_feature_dict = { + f"{k[0]}{SEPERATOR}{k[1]}": v for k, v in self._feature_to_idx_dict.items() + } + with self._model_storage.write_to(self._resource) as model_path: - rasa.utils.io.json_pickle( + rasa.shared.utils.io.dump_obj_as_json_to_file( model_path / self.FILENAME_FEATURE_TO_IDX_DICT, - self._feature_to_idx_dict, - encode_non_string_keys=True, + restructured_feature_dict, ) diff --git a/rasa/nlu/featurizers/sparse_featurizer/regex_featurizer.py b/rasa/nlu/featurizers/sparse_featurizer/regex_featurizer.py index fee53fd5b4f6..baed7f2c4852 100644 --- a/rasa/nlu/featurizers/sparse_featurizer/regex_featurizer.py +++ b/rasa/nlu/featurizers/sparse_featurizer/regex_featurizer.py @@ -1,11 +1,13 @@ from __future__ import annotations + import logging import re from typing import Any, Dict, List, Optional, Text, Tuple, Type + import numpy as np import scipy.sparse -from rasa.nlu.tokenizers.tokenizer import Tokenizer +from rasa.nlu.tokenizers.tokenizer import Tokenizer import rasa.shared.utils.io import rasa.utils.io import rasa.nlu.utils.pattern_utils as pattern_utils @@ -240,7 +242,7 @@ def load( try: with model_storage.read_from(resource) as model_dir: - patterns_file_name = model_dir / "patterns.pkl" + patterns_file_name = model_dir / "patterns.json" known_patterns = rasa.shared.utils.io.read_json_file(patterns_file_name) except (ValueError, FileNotFoundError): logger.warning( @@ -258,7 +260,7 @@ def load( def _persist(self) -> None: with self._model_storage.write_to(self._resource) as model_dir: - regex_file = model_dir / "patterns.pkl" + regex_file = model_dir / "patterns.json" rasa.shared.utils.io.dump_obj_as_json_to_file( regex_file, self.known_patterns ) diff --git a/rasa/shared/nlu/training_data/features.py b/rasa/shared/nlu/training_data/features.py index d981c1563bb0..0c5553df20c8 100644 --- a/rasa/shared/nlu/training_data/features.py +++ b/rasa/shared/nlu/training_data/features.py @@ -1,15 +1,133 @@ from __future__ import annotations -from typing import Iterable, Union, Text, Optional, List, Any, Tuple, Dict, Set + import itertools +from dataclasses import dataclass +from typing import Iterable, Union, Text, Optional, List, Any, Tuple, Dict, Set import numpy as np import scipy.sparse +from safetensors.numpy import save_file, load_file -import rasa.shared.utils.io import rasa.shared.nlu.training_data.util +import rasa.shared.utils.io from rasa.shared.nlu.constants import FEATURE_TYPE_SEQUENCE, FEATURE_TYPE_SENTENCE +@dataclass +class FeatureMetadata: + data_type: str + attribute: str + origin: Union[str, List[str]] + is_sparse: bool + shape: tuple + safetensors_key: str + + +def save_features( + features_dict: Dict[Text, List[Features]], file_name: str +) -> Dict[str, Any]: + """Save a dictionary of Features lists to disk using safetensors. + + Args: + features_dict: Dictionary mapping strings to lists of Features objects + file_name: File to save the features to + + Returns: + The metadata to reconstruct the features. + """ + # All tensors are stored in a single safetensors file + tensors_to_save = {} + # Metadata will be stored separately + metadata = {} + + for key, features_list in features_dict.items(): + feature_metadata_list = [] + + for idx, feature in enumerate(features_list): + # Create a unique key for this tensor in the safetensors file + safetensors_key = f"{key}_{idx}" + + # Convert sparse matrices to dense if needed + if feature.is_sparse(): + # For sparse matrices, use the COO format + coo = feature.features.tocoo() # type:ignore[union-attr] + # Save data, row indices and col indices separately + tensors_to_save[f"{safetensors_key}_data"] = coo.data + tensors_to_save[f"{safetensors_key}_row"] = coo.row + tensors_to_save[f"{safetensors_key}_col"] = coo.col + else: + tensors_to_save[safetensors_key] = feature.features + + # Store metadata + metadata_item = FeatureMetadata( + data_type=feature.type, + attribute=feature.attribute, + origin=feature.origin, + is_sparse=feature.is_sparse(), + shape=feature.features.shape, + safetensors_key=safetensors_key, + ) + feature_metadata_list.append(vars(metadata_item)) + + metadata[key] = feature_metadata_list + + # Save tensors + save_file(tensors_to_save, file_name) + + return metadata + + +def load_features( + filename: str, metadata: Dict[str, Any] +) -> Dict[Text, List[Features]]: + """Load Features dictionary from disk. + + Args: + filename: File name of the safetensors file. + metadata: Metadata to reconstruct the features. + + Returns: + Dictionary mapping strings to lists of Features objects + """ + # Load tensors + tensors = load_file(filename) + + # Reconstruct the features dictionary + features_dict: Dict[Text, List[Features]] = {} + + for key, feature_metadata_list in metadata.items(): + features_list = [] + + for meta in feature_metadata_list: + safetensors_key = meta["safetensors_key"] + + if meta["is_sparse"]: + # Reconstruct sparse matrix from COO format + data = tensors[f"{safetensors_key}_data"] + row = tensors[f"{safetensors_key}_row"] + col = tensors[f"{safetensors_key}_col"] + + features_matrix = scipy.sparse.coo_matrix( + (data, (row, col)), shape=tuple(meta["shape"]) + ).tocsr() # Convert back to CSR format + else: + features_matrix = tensors[safetensors_key] + + # Reconstruct Features object + features = Features( + features=features_matrix, + feature_type=meta["data_type"], + attribute=meta["attribute"], + origin=meta["origin"], + ) + + features_list.append(features) + + features_dict[key] = features_list + + return features_dict + + class Features: """Stores the features produced by any featurizer.""" diff --git a/rasa/shared/utils/io.py b/rasa/shared/utils/io.py index de2b1bc28f6c..3b13b5c18063 100644 --- a/rasa/shared/utils/io.py +++ b/rasa/shared/utils/io.py @@ -12,6 +12,7 @@ import warnings import random import string + import portalocker from ruamel import yaml as yaml diff --git a/rasa/utils/common.py b/rasa/utils/common.py index 27b754664317..f99f54042a1f 100644 --- a/rasa/utils/common.py +++ b/rasa/utils/common.py @@ -8,6 +8,7 @@ import tempfile import warnings from pathlib import Path +from socket import SOCK_DGRAM, SOCK_STREAM from types import TracebackType from typing import ( Any, @@ -24,8 +25,9 @@ Tuple, ) -from socket import SOCK_DGRAM, SOCK_STREAM import numpy as np + +import rasa.shared.utils.io import rasa.utils.io from rasa.constants import ( DEFAULT_LOG_LEVEL_LIBRARIES, @@ -36,7 +38,6 @@ ) from rasa.shared.constants import DEFAULT_LOG_LEVEL, ENV_LOG_LEVEL, TCP_PROTOCOL from rasa.shared.exceptions import RasaException -import rasa.shared.utils.io logger = logging.getLogger(__name__) @@ -153,7 +154,7 @@ def configure_logging_from_file(logging_config_file: Text) -> None: try: logging.config.dictConfig(logging_config_dict) except (ValueError, TypeError, AttributeError, ImportError) as e: - logging.debug( + logger.debug( f"The logging config file {logging_config_file} could not " f"be applied because it failed validation against " f"the built-in Python logging schema. " diff --git a/rasa/utils/endpoints.py b/rasa/utils/endpoints.py index 5e1032778e6b..31d1ea7228bc 100644 --- a/rasa/utils/endpoints.py +++ b/rasa/utils/endpoints.py @@ -1,8 +1,6 @@ import ssl -from functools import cached_property import aiohttp -import logging import os from aiohttp.client_exceptions import ContentTypeError from sanic.request import Request @@ -11,10 +9,11 @@ from rasa.shared.exceptions import FileNotFoundException import rasa.shared.utils.io import rasa.utils.io +import structlog from rasa.core.constants import DEFAULT_REQUEST_TIMEOUT -logger = logging.getLogger(__name__) +structlogger = structlog.get_logger() def read_endpoint_config( @@ -32,9 +31,13 @@ def read_endpoint_config( return EndpointConfig.from_dict(content[endpoint_type]) except FileNotFoundError: - logger.error( - "Failed to read endpoint configuration " - "from {}. No such file.".format(os.path.abspath(filename)) + structlogger.error( + "endpoint.read.failed_no_such_file", + filename=os.path.abspath(filename), + event_info=( + "Failed to read endpoint configuration file - " + "the file was not found." + ), ) return None @@ -56,9 +59,13 @@ def concat_url(base: Text, subpath: Optional[Text]) -> Text: """ if not subpath: if base.endswith("/"): - logger.debug( - f"The URL '{base}' has a trailing slash. Please make sure the " - f"target server supports trailing slashes for this endpoint." + structlogger.debug( + "endpoint.concat_url.trailing_slash", + url=base, + event_info=( + "The URL has a trailing slash. Please make sure the " + "target server supports trailing slashes for this endpoint." + ), ) return base @@ -95,7 +102,6 @@ def __init__( self.cafile = cafile self.kwargs = kwargs - @cached_property def session(self) -> aiohttp.ClientSession: """Creates and returns a configured aiohttp client session.""" # create authentication parameters @@ -164,23 +170,26 @@ async def request( f"'{os.path.abspath(self.cafile)}' does not exist." ) from e - async with self.session.request( - method, - url, - headers=headers, - params=self.combine_parameters(kwargs), - compress=compress, - ssl=sslcontext, - **kwargs, - ) as response: - if response.status >= 400: - raise ClientResponseError( - response.status, response.reason, await response.content.read() - ) - try: - return await response.json() - except ContentTypeError: - return None + async with self.session() as session: + async with session.request( + method, + url, + headers=headers, + params=self.combine_parameters(kwargs), + compress=compress, + ssl=sslcontext, + **kwargs, + ) as response: + if response.status >= 400: + raise ClientResponseError( + response.status, + response.reason, + await response.content.read(), + ) + try: + return await response.json() + except ContentTypeError: + return None @classmethod def from_dict(cls, data: Dict[Text, Any]) -> "EndpointConfig": @@ -263,7 +272,7 @@ def float_arg( try: return float(str(arg)) except (ValueError, TypeError): - logger.warning(f"Failed to convert '{arg}' to float.") + structlogger.warning("endpoint.float_arg.convert_failed", arg=arg, key=key) return default @@ -291,5 +300,6 @@ def int_arg( try: return int(str(arg)) except (ValueError, TypeError): - logger.warning(f"Failed to convert '{arg}' to int.") + + structlogger.warning("endpoint.int_arg.convert_failed", arg=arg, key=key) return default diff --git a/rasa/utils/io.py b/rasa/utils/io.py index 3388ef98b049..da0800c61f0a 100644 --- a/rasa/utils/io.py +++ b/rasa/utils/io.py @@ -2,13 +2,13 @@ import filecmp import logging import os -import pickle +import re import tempfile import warnings -import re from asyncio import AbstractEventLoop from pathlib import Path -from typing import Text, Any, Union, List, Type, Callable, TYPE_CHECKING, Pattern +from typing import Text, Any, List, Type, Callable, TYPE_CHECKING, Pattern + from typing_extensions import Protocol import rasa.shared.constants @@ -81,29 +81,6 @@ def enable_async_loop_debugging( return event_loop -def pickle_dump(filename: Union[Text, Path], obj: Any) -> None: - """Saves object to file. - - Args: - filename: the filename to save the object to - obj: the object to store - """ - with open(filename, "wb") as f: - pickle.dump(obj, f) - - -def pickle_load(filename: Union[Text, Path]) -> Any: - """Loads an object from a file. - - Args: - filename: the filename to load the object from - - Returns: the loaded object - """ - with open(filename, "rb") as f: - return pickle.load(f) - - def create_temporary_file(data: Any, suffix: Text = "", mode: Text = "w+") -> Text: """Creates a tempfile.NamedTemporaryFile object for data.""" encoding = None if "b" in mode else rasa.shared.utils.io.DEFAULT_ENCODING @@ -124,7 +101,6 @@ def create_temporary_directory() -> Text: def create_path(file_path: Text) -> None: """Makes sure all directories in the 'file_path' exists.""" - parent_dir = os.path.dirname(os.path.abspath(file_path)) if not os.path.exists(parent_dir): os.makedirs(parent_dir) @@ -160,8 +136,8 @@ def create_validator( function: Callable[[Text], bool], error_message: Text ) -> Type["Validator"]: """Helper method to create `Validator` classes from callable functions. Should be - removed when questionary supports `Validator` objects.""" - + removed when questionary supports `Validator` objects. + """ from prompt_toolkit.validation import Validator, ValidationError from prompt_toolkit.document import Document @@ -175,48 +151,6 @@ def validate(document: Document) -> None: return FunctionValidator -def json_unpickle( - file_name: Union[Text, Path], encode_non_string_keys: bool = False -) -> Any: - """Unpickle an object from file using json. - - Args: - file_name: the file to load the object from - encode_non_string_keys: If set to `True` then jsonpickle will encode non-string - dictionary keys instead of coercing them into strings via `repr()`. - - Returns: the object - """ - import jsonpickle.ext.numpy as jsonpickle_numpy - import jsonpickle - - jsonpickle_numpy.register_handlers() - - file_content = rasa.shared.utils.io.read_file(file_name) - return jsonpickle.loads(file_content, keys=encode_non_string_keys) - - -def json_pickle( - file_name: Union[Text, Path], obj: Any, encode_non_string_keys: bool = False -) -> None: - """Pickle an object to a file using json. - - Args: - file_name: the file to store the object to - obj: the object to store - encode_non_string_keys: If set to `True` then jsonpickle will encode non-string - dictionary keys instead of coercing them into strings via `repr()`. - """ - import jsonpickle.ext.numpy as jsonpickle_numpy - import jsonpickle - - jsonpickle_numpy.register_handlers() - - rasa.shared.utils.io.write_text_file( - jsonpickle.dumps(obj, keys=encode_non_string_keys), file_name - ) - - def get_emoji_regex() -> Pattern: """Returns regex to identify emojis.""" return re.compile( diff --git a/rasa/utils/tensorflow/constants.py b/rasa/utils/tensorflow/constants.py index 047db9878c67..39d5ea6d0560 100644 --- a/rasa/utils/tensorflow/constants.py +++ b/rasa/utils/tensorflow/constants.py @@ -113,3 +113,4 @@ USE_GPU = "use_gpu" RUN_EAGERLY = "run_eagerly" +DROP_SMALL_LAST_BATCH = "drop_small_last_batch" diff --git a/rasa/utils/tensorflow/data_generator.py b/rasa/utils/tensorflow/data_generator.py index a696f607c026..e54b95dad335 100644 --- a/rasa/utils/tensorflow/data_generator.py +++ b/rasa/utils/tensorflow/data_generator.py @@ -344,6 +344,7 @@ def __init__( epochs: int = 1, batch_strategy: Text = SEQUENCE, shuffle: bool = True, + drop_small_last_batch: bool = False, ): """Initializes the increasing batch size data generator. @@ -353,6 +354,8 @@ def __init__( epochs: The total number of epochs. batch_strategy: The batch strategy. shuffle: If 'True', data will be shuffled. + drop_small_last_batch: if 'True', the last batch in an epoch will be dropped + if it has less examples than half the batch size """ super().__init__(model_data, batch_size, batch_strategy, shuffle) @@ -370,6 +373,7 @@ def __init__( self._current_batch_size = 0 # create separate data variable that will store modified data for each batch self._data: Data = {} + self.drop_small_last_batch = drop_small_last_batch self.on_epoch_end() def __len__(self) -> int: @@ -381,11 +385,16 @@ def __len__(self) -> int: # data was rebalanced, so need to recalculate number of examples num_examples = self.model_data.number_of_examples(self._data) batch_size = self._current_batch_size - # keep last batch only if it has at least half a batch size of examples - last_batch_half_full = num_examples % batch_size >= math.ceil(batch_size / 2) - num_batches = num_examples // batch_size + int(last_batch_half_full) - # Return at least 1 if there is an example - return max(num_batches, int(num_examples > 0)) + if self.drop_small_last_batch: + # keep last batch only if it has at least half a batch size of examples + last_batch_half_full = num_examples % batch_size >= math.ceil( + batch_size / 2 + ) + num_batches = num_examples // batch_size + int(last_batch_half_full) + # Return at least 1 if there is an example + return max(num_batches, int(num_examples > 0)) + else: + return num_examples // batch_size + int(num_examples % batch_size > 0) def __getitem__(self, index: int) -> Tuple[Any, Any]: """Gets batch at position `index`. diff --git a/rasa/utils/tensorflow/feature_array.py b/rasa/utils/tensorflow/feature_array.py new file mode 100644 index 000000000000..9af50e3ceb66 --- /dev/null +++ b/rasa/utils/tensorflow/feature_array.py @@ -0,0 +1,370 @@ +from typing import Dict, Any, List, Tuple, Optional, Union + +import numpy as np +import scipy.sparse +from safetensors.numpy import load_file +from safetensors.numpy import save_file + +import rasa.shared.utils.io + + +def _recursive_serialize( + array: Any, prefix: str, data_dict: Dict[str, Any], metadata: List[Dict[str, Any]] +) -> None: + """Recursively serialize arrays and matrices for high dimensional data.""" + if isinstance(array, np.ndarray) and array.ndim <= 2: + data_key = f"{prefix}_array" + data_dict[data_key] = array + metadata.append({"type": "dense", "key": data_key, "shape": array.shape}) + + elif isinstance(array, list) and all([isinstance(v, float) for v in array]): + data_key = f"{prefix}_list" + data_dict[data_key] = np.array(array, dtype=np.float32) + metadata.append({"type": "list", "key": data_key}) + + elif isinstance(array, list) and all([isinstance(v, int) for v in array]): + data_key = f"{prefix}_list" + data_dict[data_key] = np.array(array, dtype=np.int64) + metadata.append({"type": "list", "key": data_key}) + + elif isinstance(array, scipy.sparse.spmatrix): + data_key_data = f"{prefix}_data" + data_key_row = f"{prefix}_row" + data_key_col = f"{prefix}_col" + array = array.tocoo() + data_dict.update( + { + data_key_data: array.data, + data_key_row: array.row, + data_key_col: array.col, + } + ) + metadata.append({"type": "sparse", "key": prefix, "shape": array.shape}) + + elif isinstance(array, list) or isinstance(array, np.ndarray): + group_metadata = {"type": "group", "subcomponents": []} + for idx, item in enumerate(array): + new_prefix = f"{prefix}_{idx}" + _recursive_serialize( + item, new_prefix, data_dict, group_metadata["subcomponents"] + ) + metadata.append(group_metadata) + + +def _serialize_nested_data( + nested_data: Dict[str, Dict[str, List["FeatureArray"]]], + prefix: str, + data_dict: Dict[str, np.ndarray], + metadata: List[Dict[str, Union[str, List]]], +) -> None: + """Handle serialization across dictionary and list levels.""" + for outer_key, inner_dict in nested_data.items(): + inner_metadata = {"key": outer_key, "components": []} + + for inner_key, feature_arrays in inner_dict.items(): + array_metadata = { + "key": inner_key, + "number_of_dimensions": feature_arrays[0].number_of_dimensions, + "features": [], + } + + for idx, feature_array in enumerate(feature_arrays): + feature_prefix = f"{prefix}_{outer_key}_{inner_key}_{idx}" + _recursive_serialize( + feature_array.tolist(), + feature_prefix, + data_dict, + array_metadata["features"], + ) + + inner_metadata["components"].append( # type:ignore[attr-defined] + array_metadata + ) + + metadata.append(inner_metadata) + + +def serialize_nested_feature_arrays( + nested_feature_array: Dict[str, Dict[str, List["FeatureArray"]]], + data_filename: str, + metadata_filename: str, +) -> None: + data_dict: Dict[str, np.ndarray] = {} + metadata: List[Dict[str, Union[str, List]]] = [] + + _serialize_nested_data(nested_feature_array, "component", data_dict, metadata) + + # Save serialized data and metadata + save_file(data_dict, data_filename) + rasa.shared.utils.io.dump_obj_as_json_to_file(metadata_filename, metadata) + + +def _recursive_deserialize( + metadata: List[Dict[str, Any]], data: Dict[str, Any] +) -> List[Any]: + """Recursively deserialize arrays and matrices for high dimensional data.""" + result = [] + + for item in metadata: + if item["type"] == "dense": + key = item["key"] + array = np.asarray(data[key]).reshape(item["shape"]) + result.append(array) + + elif item["type"] == "list": + key = item["key"] + result.append(list(data[key])) + + elif item["type"] == "sparse": + data_vals = data[f"{item['key']}_data"] + row_vals = data[f"{item['key']}_row"] + col_vals = data[f"{item['key']}_col"] + sparse_matrix = scipy.sparse.coo_matrix( + (data_vals, (row_vals, col_vals)), shape=item["shape"] + ) + result.append(sparse_matrix) + elif item["type"] == "group": + sublist = _recursive_deserialize(item["subcomponents"], data) + result.append(sublist) + + return result + + +def _deserialize_nested_data( + metadata: List[Dict[str, Any]], data_dict: Dict[str, Any] +) -> Dict[str, Dict[str, List["FeatureArray"]]]: + """Handle deserialization across all dictionary and list levels.""" + result: Dict[str, Dict[str, List["FeatureArray"]]] = {} + + for outer_item in metadata: + outer_key = outer_item["key"] + result[outer_key] = {} + + for inner_item in outer_item["components"]: + inner_key = inner_item["key"] + feature_arrays = [] + + # Reconstruct the list of FeatureArrays + for feature_item in inner_item["features"]: + # Reconstruct the list of FeatureArrays + feature_array_data = _recursive_deserialize([feature_item], data_dict) + # Prepare the input for the FeatureArray; + # ensure it is np.ndarray compatible + input_array = np.array(feature_array_data[0], dtype=object) + feature_array = FeatureArray( + input_array, inner_item["number_of_dimensions"] + ) + feature_arrays.append(feature_array) + + result[outer_key][inner_key] = feature_arrays + + return result + + +def deserialize_nested_feature_arrays( + data_filename: str, metadata_filename: str +) -> Dict[str, Dict[str, List["FeatureArray"]]]: + metadata = rasa.shared.utils.io.read_json_file(metadata_filename) + data_dict = load_file(data_filename) + + return _deserialize_nested_data(metadata, data_dict) + + +class FeatureArray(np.ndarray): + """Stores any kind of features ready to be used by a RasaModel. + + Next to the input numpy array of features, it also received the number of + dimensions of the features. + As our features can have 1 to 4 dimensions we might have different number of numpy + arrays stacked. The number of dimensions helps us to figure out how to handle this + particular feature array. Also, it is automatically determined whether the feature + array is sparse or not and the number of units is determined as well. + + Subclassing np.array: https://numpy.org/doc/stable/user/basics.subclassing.html + """ + + def __new__( + cls, input_array: np.ndarray, number_of_dimensions: int + ) -> "FeatureArray": + """Create and return a new object. See help(type) for accurate signature.""" + FeatureArray._validate_number_of_dimensions(number_of_dimensions, input_array) + + feature_array = np.asarray(input_array).view(cls) + + if number_of_dimensions <= 2: + feature_array.units = input_array.shape[-1] + feature_array.is_sparse = isinstance(input_array[0], scipy.sparse.spmatrix) + elif number_of_dimensions == 3: + feature_array.units = input_array[0].shape[-1] + feature_array.is_sparse = isinstance(input_array[0], scipy.sparse.spmatrix) + elif number_of_dimensions == 4: + feature_array.units = input_array[0][0].shape[-1] + feature_array.is_sparse = isinstance( + input_array[0][0], scipy.sparse.spmatrix + ) + else: + raise ValueError( + f"Number of dimensions '{number_of_dimensions}' currently not " + f"supported." + ) + + feature_array.number_of_dimensions = number_of_dimensions + + return feature_array + + def __init__( + self, input_array: Any, number_of_dimensions: int, **kwargs: Any + ) -> None: + """Initialize. FeatureArray. + + Needed in order to avoid 'Invalid keyword argument number_of_dimensions + to function FeatureArray.__init__ ' + Args: + input_array: the array that contains features + number_of_dimensions: number of dimensions in input_array + """ + super().__init__(**kwargs) + self.number_of_dimensions = number_of_dimensions + + def __array_finalize__(self, obj: Optional[np.ndarray]) -> None: + """This method is called when the system allocates a new array from obj. + + Args: + obj: A subclass (subtype) of ndarray. + """ + if obj is None: + return + + self.units = getattr(obj, "units", None) + self.number_of_dimensions = getattr( + obj, "number_of_dimensions", None + ) # type: ignore[assignment] + self.is_sparse = getattr(obj, "is_sparse", None) + + default_attributes = { + "units": self.units, + "number_of_dimensions": self.number_of_dimensions, + "is_spare": self.is_sparse, + } + self.__dict__.update(default_attributes) + + # pytype: disable=attribute-error + def __array_ufunc__( + self, ufunc: Any, method: str, *inputs: Any, **kwargs: Any + ) -> Any: + """Overwrite this method as we are subclassing numpy array. + + Args: + ufunc: The ufunc object that was called. + method: A string indicating which Ufunc method was called + (one of "__call__", "reduce", "reduceat", "accumulate", "outer", + "inner"). + *inputs: A tuple of the input arguments to the ufunc. + **kwargs: Any additional arguments + + Returns: + The result of the operation. + """ + f = { + "reduce": ufunc.reduce, + "accumulate": ufunc.accumulate, + "reduceat": ufunc.reduceat, + "outer": ufunc.outer, + "at": ufunc.at, + "__call__": ufunc, + } + # convert the inputs to np.ndarray to prevent recursion, call the function, + # then cast it back as FeatureArray + output = FeatureArray( + f[method](*(i.view(np.ndarray) for i in inputs), **kwargs), + number_of_dimensions=kwargs["number_of_dimensions"], + ) + output.__dict__ = self.__dict__ # carry forward attributes + return output + + def __reduce__(self) -> Tuple[Any, Any, Any]: + """Needed in order to pickle this object. + + Returns: + A tuple. + """ + pickled_state = super(FeatureArray, self).__reduce__() + if isinstance(pickled_state, str): + raise TypeError("np array __reduce__ returned string instead of tuple.") + new_state = pickled_state[2] + ( + self.number_of_dimensions, + self.is_sparse, + self.units, + ) + return pickled_state[0], pickled_state[1], new_state + + def __setstate__(self, state: Any, **kwargs: Any) -> None: + """Sets the state. + + Args: + state: The state argument must be a sequence that contains the following + elements version, shape, dtype, isFortan, rawdata. + **kwargs: Any additional parameter + """ + # Needed in order to load the object + self.number_of_dimensions = state[-3] + self.is_sparse = state[-2] + self.units = state[-1] + super(FeatureArray, self).__setstate__(state[0:-3], **kwargs) + + # pytype: enable=attribute-error + + @staticmethod + def _validate_number_of_dimensions( + number_of_dimensions: int, input_array: np.ndarray + ) -> None: + """Validates if the input array has given number of dimensions. + + Args: + number_of_dimensions: number of dimensions + input_array: input array + + Raises: ValueError in case the dimensions do not match + """ + # when loading the feature arrays from disk, the shape represents + # the correct number of dimensions + if len(input_array.shape) == number_of_dimensions: + return + + _sub_array = input_array + dim = 0 + # Go number_of_dimensions into the given input_array + for i in range(1, number_of_dimensions + 1): + _sub_array = _sub_array[0] + if isinstance(_sub_array, scipy.sparse.spmatrix): + dim = i + break + if isinstance(_sub_array, np.ndarray) and _sub_array.shape[0] == 0: + # sequence dimension is 0, we are dealing with "fake" features + dim = i + break + + # If the resulting sub_array is sparse, the remaining number of dimensions + # should be at least 2 + if isinstance(_sub_array, scipy.sparse.spmatrix): + if dim > 2: + raise ValueError( + f"Given number of dimensions '{number_of_dimensions}' does not " + f"match dimensions of given input array: {input_array}." + ) + elif isinstance(_sub_array, np.ndarray) and _sub_array.shape[0] == 0: + # sequence dimension is 0, we are dealing with "fake" features, + # but they should be of dim 2 + if dim > 2: + raise ValueError( + f"Given number of dimensions '{number_of_dimensions}' does not " + f"match dimensions of given input array: {input_array}." + ) + # If the resulting sub_array is dense, the sub_array should be a single number + elif not np.issubdtype(type(_sub_array), np.integer) and not isinstance( + _sub_array, (np.float32, np.float64) + ): + raise ValueError( + f"Given number of dimensions '{number_of_dimensions}' does not match " + f"dimensions of given input array: {input_array}." + ) diff --git a/rasa/utils/tensorflow/model_data.py b/rasa/utils/tensorflow/model_data.py index 128ff6cbd575..393756972305 100644 --- a/rasa/utils/tensorflow/model_data.py +++ b/rasa/utils/tensorflow/model_data.py @@ -20,6 +20,8 @@ import scipy.sparse from sklearn.model_selection import train_test_split +from rasa.utils.tensorflow.feature_array import FeatureArray + logger = logging.getLogger(__name__) @@ -37,199 +39,6 @@ def ragged_array_to_ndarray(ragged_array: Iterable[np.ndarray]) -> np.ndarray: return np.array(ragged_array, dtype=object) -class FeatureArray(np.ndarray): - """Stores any kind of features ready to be used by a RasaModel. - - Next to the input numpy array of features, it also received the number of - dimensions of the features. - As our features can have 1 to 4 dimensions we might have different number of numpy - arrays stacked. The number of dimensions helps us to figure out how to handle this - particular feature array. Also, it is automatically determined whether the feature - array is sparse or not and the number of units is determined as well. - - Subclassing np.array: https://numpy.org/doc/stable/user/basics.subclassing.html - """ - - def __new__( - cls, input_array: np.ndarray, number_of_dimensions: int - ) -> "FeatureArray": - """Create and return a new object. See help(type) for accurate signature.""" - FeatureArray._validate_number_of_dimensions(number_of_dimensions, input_array) - - feature_array = np.asarray(input_array).view(cls) - - if number_of_dimensions <= 2: - feature_array.units = input_array.shape[-1] - feature_array.is_sparse = isinstance(input_array[0], scipy.sparse.spmatrix) - elif number_of_dimensions == 3: - feature_array.units = input_array[0].shape[-1] - feature_array.is_sparse = isinstance(input_array[0], scipy.sparse.spmatrix) - elif number_of_dimensions == 4: - feature_array.units = input_array[0][0].shape[-1] - feature_array.is_sparse = isinstance( - input_array[0][0], scipy.sparse.spmatrix - ) - else: - raise ValueError( - f"Number of dimensions '{number_of_dimensions}' currently not " - f"supported." - ) - - feature_array.number_of_dimensions = number_of_dimensions - - return feature_array - - def __init__( - self, input_array: Any, number_of_dimensions: int, **kwargs: Any - ) -> None: - """Initialize. FeatureArray. - - Needed in order to avoid 'Invalid keyword argument number_of_dimensions - to function FeatureArray.__init__ ' - Args: - input_array: the array that contains features - number_of_dimensions: number of dimensions in input_array - """ - super().__init__(**kwargs) - self.number_of_dimensions = number_of_dimensions - - def __array_finalize__(self, obj: Optional[np.ndarray]) -> None: - """This method is called when the system allocates a new array from obj. - - Args: - obj: A subclass (subtype) of ndarray. - """ - if obj is None: - return - - self.units = getattr(obj, "units", None) - self.number_of_dimensions = getattr(obj, "number_of_dimensions", None) # type: ignore[assignment] # noqa:E501 - self.is_sparse = getattr(obj, "is_sparse", None) - - default_attributes = { - "units": self.units, - "number_of_dimensions": self.number_of_dimensions, - "is_spare": self.is_sparse, - } - self.__dict__.update(default_attributes) - - # pytype: disable=attribute-error - def __array_ufunc__( - self, ufunc: Any, method: Text, *inputs: Any, **kwargs: Any - ) -> Any: - """Overwrite this method as we are subclassing numpy array. - - Args: - ufunc: The ufunc object that was called. - method: A string indicating which Ufunc method was called - (one of "__call__", "reduce", "reduceat", "accumulate", "outer", - "inner"). - *inputs: A tuple of the input arguments to the ufunc. - **kwargs: Any additional arguments - - Returns: - The result of the operation. - """ - f = { - "reduce": ufunc.reduce, - "accumulate": ufunc.accumulate, - "reduceat": ufunc.reduceat, - "outer": ufunc.outer, - "at": ufunc.at, - "__call__": ufunc, - } - # convert the inputs to np.ndarray to prevent recursion, call the function, - # then cast it back as FeatureArray - output = FeatureArray( - f[method](*(i.view(np.ndarray) for i in inputs), **kwargs), - number_of_dimensions=kwargs["number_of_dimensions"], - ) - output.__dict__ = self.__dict__ # carry forward attributes - return output - - def __reduce__(self) -> Tuple[Any, Any, Any]: - """Needed in order to pickle this object. - - Returns: - A tuple. - """ - pickled_state = super(FeatureArray, self).__reduce__() - if isinstance(pickled_state, str): - raise TypeError("np array __reduce__ returned string instead of tuple.") - new_state = pickled_state[2] + ( - self.number_of_dimensions, - self.is_sparse, - self.units, - ) - return pickled_state[0], pickled_state[1], new_state - - def __setstate__(self, state: Any, **kwargs: Any) -> None: - """Sets the state. - - Args: - state: The state argument must be a sequence that contains the following - elements version, shape, dtype, isFortan, rawdata. - **kwargs: Any additional parameter - """ - # Needed in order to load the object - self.number_of_dimensions = state[-3] - self.is_sparse = state[-2] - self.units = state[-1] - super(FeatureArray, self).__setstate__(state[0:-3], **kwargs) - - # pytype: enable=attribute-error - - @staticmethod - def _validate_number_of_dimensions( - number_of_dimensions: int, input_array: np.ndarray - ) -> None: - """Validates if the the input array has given number of dimensions. - - Args: - number_of_dimensions: number of dimensions - input_array: input array - - Raises: ValueError in case the dimensions do not match - """ - _sub_array = input_array - dim = 0 - # Go number_of_dimensions into the given input_array - for i in range(1, number_of_dimensions + 1): - _sub_array = _sub_array[0] - if isinstance(_sub_array, scipy.sparse.spmatrix): - dim = i - break - if isinstance(_sub_array, np.ndarray) and _sub_array.shape[0] == 0: - # sequence dimension is 0, we are dealing with "fake" features - dim = i - break - - # If the resulting sub_array is sparse, the remaining number of dimensions - # should be at least 2 - if isinstance(_sub_array, scipy.sparse.spmatrix): - if dim > 2: - raise ValueError( - f"Given number of dimensions '{number_of_dimensions}' does not " - f"match dimensions of given input array: {input_array}." - ) - elif isinstance(_sub_array, np.ndarray) and _sub_array.shape[0] == 0: - # sequence dimension is 0, we are dealing with "fake" features, - # but they should be of dim 2 - if dim > 2: - raise ValueError( - f"Given number of dimensions '{number_of_dimensions}' does not " - f"match dimensions of given input array: {input_array}." - ) - # If the resulting sub_array is dense, the sub_array should be a single number - elif not np.issubdtype(type(_sub_array), np.integer) and not isinstance( - _sub_array, (np.float32, np.float64) - ): - raise ValueError( - f"Given number of dimensions '{number_of_dimensions}' does not match " - f"dimensions of given input array: {input_array}." - ) - - class FeatureSignature(NamedTuple): """Signature of feature arrays. @@ -270,8 +79,7 @@ def __init__( label_sub_key: Optional[Text] = None, data: Optional[Data] = None, ) -> None: - """ - Initializes the RasaModelData object. + """Initializes the RasaModelData object. Args: label_key: the key of a label used for balancing, etc. diff --git a/rasa/utils/train_utils.py b/rasa/utils/train_utils.py index 36de0370d210..764507d7e39d 100644 --- a/rasa/utils/train_utils.py +++ b/rasa/utils/train_utils.py @@ -302,6 +302,7 @@ def create_data_generators( eval_num_examples: int = 0, random_seed: Optional[int] = None, shuffle: bool = True, + drop_small_last_batch: bool = False, ) -> Tuple[RasaBatchDataGenerator, Optional[RasaBatchDataGenerator]]: """Create data generators for train and optional validation data. @@ -313,6 +314,8 @@ def create_data_generators( eval_num_examples: Number of examples to use for validation data. random_seed: The random seed. shuffle: Whether to shuffle data inside the data generator. + drop_small_last_batch: whether to drop the last batch if it has fewer than half + a batch size of examples Returns: The training data generator and optional validation data generator. @@ -328,6 +331,7 @@ def create_data_generators( epochs=epochs, batch_strategy=batch_strategy, shuffle=shuffle, + drop_small_last_batch=drop_small_last_batch, ) data_generator = RasaBatchDataGenerator( @@ -336,6 +340,7 @@ def create_data_generators( epochs=epochs, batch_strategy=batch_strategy, shuffle=shuffle, + drop_small_last_batch=drop_small_last_batch, ) return data_generator, validation_data_generator diff --git a/rasa/version.py b/rasa/version.py index 7f4d3334548b..3b9621033a5d 100644 --- a/rasa/version.py +++ b/rasa/version.py @@ -1,3 +1,3 @@ # this file will automatically be changed, # do not add anything but the version number here! -__version__ = "3.6.13" +__version__ = "3.6.21" diff --git a/scripts/ping_slack_about_package_release.sh b/scripts/ping_slack_about_package_release.sh old mode 100755 new mode 100644 index ef97ead7a178..b8ee68b4b6cd --- a/scripts/ping_slack_about_package_release.sh +++ b/scripts/ping_slack_about_package_release.sh @@ -7,4 +7,3 @@ if [[ ${GITHUB_TAG} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then --data "{\"text\":\"๐Ÿ’ฅ New *Rasa Open Source* version ${GITHUB_TAG} has been released! https://github.com/RasaHQ/rasa/releases/tag/${GITHUB_TAG}\"}" \ "https://hooks.slack.com/services/T0GHWFTS8/BMTQQL47K/${SLACK_WEBHOOK_TOKEN}" fi - diff --git a/scripts/release.py b/scripts/release.py index d1ac98325f80..068bd14e5fd2 100644 --- a/scripts/release.py +++ b/scripts/release.py @@ -30,6 +30,10 @@ RELEASE_BRANCH_PATTERN = re.compile(r"^\d+\.\d+\.x$") +PUBLIC_REMOTE = "public" +DEFAULT_REMOTE = "origin" +FIRST_CALM_VERSION = "3.7.0" + def create_argument_parser() -> argparse.ArgumentParser: """Parse all the command line arguments for the release script.""" @@ -247,9 +251,9 @@ def create_commit(version: Version) -> None: check_call(["git", "commit", "-m", f"prepared release of version {version}"]) -def push_changes() -> None: - """Pushes the current branch to origin.""" - check_call(["git", "push", "origin", "HEAD"]) +def push_changes(remote: str = DEFAULT_REMOTE) -> None: + """Pushes the current branch to the specified remote.""" + check_call(["git", "push", remote, "HEAD"]) def ensure_clean_git() -> None: @@ -337,10 +341,11 @@ def main(args: argparse.Namespace) -> None: # never update changelog on a prerelease version generate_changelog(version) + remote = PUBLIC_REMOTE if str(version) < FIRST_CALM_VERSION else DEFAULT_REMOTE # alpha workflow on feature branch when a version bump is required if version.is_alpha and not git_current_branch_is_main_or_release(): create_commit(version) - push_changes() + push_changes(remote) print_done_message_same_branch(version) else: @@ -348,7 +353,7 @@ def main(args: argparse.Namespace) -> None: branch = create_release_branch(version) create_commit(version) - push_changes() + push_changes(remote) print_done_message(branch, base, version) diff --git a/tests/core/featurizers/test_tracker_featurizer.py b/tests/core/featurizers/test_tracker_featurizer.py index 99ffea6e9641..20a0a08f558d 100644 --- a/tests/core/featurizers/test_tracker_featurizer.py +++ b/tests/core/featurizers/test_tracker_featurizer.py @@ -34,7 +34,25 @@ def test_fail_to_load_non_existent_featurizer(): assert TrackerFeaturizer.load("non_existent_class") is None -def test_persist_and_load_tracker_featurizer(tmp_path: Text, moodbot_domain: Domain): +def test_persist_and_load_full_dialogue_tracker_featurizer( + tmp_path: Text, moodbot_domain: Domain +): + state_featurizer = SingleStateFeaturizer() + state_featurizer.prepare_for_training(moodbot_domain) + tracker_featurizer = FullDialogueTrackerFeaturizer(state_featurizer) + + tracker_featurizer.persist(tmp_path) + + loaded_tracker_featurizer = TrackerFeaturizer.load(tmp_path) + + assert loaded_tracker_featurizer is not None + assert loaded_tracker_featurizer.state_featurizer is not None + assert loaded_tracker_featurizer.to_dict() == tracker_featurizer.to_dict() + + +def test_persist_and_load_max_history_tracker_featurizer( + tmp_path: Text, moodbot_domain: Domain +): state_featurizer = SingleStateFeaturizer() state_featurizer.prepare_for_training(moodbot_domain) tracker_featurizer = MaxHistoryTrackerFeaturizer(state_featurizer) @@ -45,6 +63,23 @@ def test_persist_and_load_tracker_featurizer(tmp_path: Text, moodbot_domain: Dom assert loaded_tracker_featurizer is not None assert loaded_tracker_featurizer.state_featurizer is not None + assert loaded_tracker_featurizer.to_dict() == tracker_featurizer.to_dict() + + +def test_persist_and_load_intent_max_history_tracker_featurizer( + tmp_path: Text, moodbot_domain: Domain +): + state_featurizer = SingleStateFeaturizer() + state_featurizer.prepare_for_training(moodbot_domain) + tracker_featurizer = IntentMaxHistoryTrackerFeaturizer(state_featurizer) + + tracker_featurizer.persist(tmp_path) + + loaded_tracker_featurizer = TrackerFeaturizer.load(tmp_path) + + assert loaded_tracker_featurizer is not None + assert loaded_tracker_featurizer.state_featurizer is not None + assert loaded_tracker_featurizer.to_dict() == tracker_featurizer.to_dict() def test_convert_action_labels_to_ids(domain: Domain): @@ -127,7 +162,6 @@ def compare_featurized_states( """Compares two lists of featurized states and returns True if they are identical and False otherwise. """ - if len(states1) != len(states2): return False diff --git a/tests/core/test_exporter.py b/tests/core/test_exporter.py index b74849c6bb33..0ea78f4b397f 100644 --- a/tests/core/test_exporter.py +++ b/tests/core/test_exporter.py @@ -73,9 +73,9 @@ async def test_fetch_events_within_time_range(): conversation_ids = ["some-id", "another-id"] # prepare events from different senders and different timestamps - event_1 = random_user_uttered_event(3) - event_2 = random_user_uttered_event(2) - event_3 = random_user_uttered_event(1) + event_1 = random_user_uttered_event(1) + event_2 = random_user_uttered_event(3) + event_3 = random_user_uttered_event(2) events = {conversation_ids[0]: [event_1, event_2], conversation_ids[1]: [event_3]} def _get_tracker(conversation_id: Text) -> DialogueStateTracker: diff --git a/tests/core/test_processor.py b/tests/core/test_processor.py index 392d85c29745..d0581b1800ef 100644 --- a/tests/core/test_processor.py +++ b/tests/core/test_processor.py @@ -70,7 +70,12 @@ from rasa.core.http_interpreter import RasaNLUHttpInterpreter from rasa.core.processor import MessageProcessor from rasa.shared.core.trackers import DialogueStateTracker -from rasa.shared.nlu.constants import INTENT_NAME_KEY, METADATA_MODEL_ID +from rasa.shared.nlu.constants import ( + INTENT, + INTENT_NAME_KEY, + FULL_RETRIEVAL_INTENT_NAME_KEY, + METADATA_MODEL_ID, +) from rasa.shared.nlu.training_data.message import Message from rasa.utils.endpoints import EndpointConfig from rasa.shared.core.constants import ( @@ -1928,3 +1933,60 @@ async def test_run_anonymization_pipeline_mocked_pipeline( await processor.run_anonymization_pipeline(tracker) event_diff.assert_called_once() + + +async def test_update_full_retrieval_intent( + default_processor: MessageProcessor, +) -> None: + parse_data = { + "text": "I like sunny days in berlin", + "intent": {"name": "chitchat", "confidence": 0.9}, + "entities": [], + "response_selector": { + "all_retrieval_intents": ["faq", "chitchat"], + "faq": { + "response": { + "responses": [{"text": "Our return policy lasts 30 days."}], + "confidence": 1.0, + "intent_response_key": "faq/what_is_return_policy", + "utter_action": "utter_faq/what_is_return_policy", + }, + "ranking": [ + { + "confidence": 1.0, + "intent_response_key": "faq/what_is_return_policy", + }, + { + "confidence": 2.3378809862799945e-19, + "intent_response_key": "faq/how_can_i_track_my_order", + }, + ], + }, + "chitchat": { + "response": { + "responses": [ + { + "text": "The sun is out today! Isn't that great?", + }, + ], + "confidence": 1.0, + "intent_response_key": "chitchat/ask_weather", + "utter_action": "utter_chitchat/ask_weather", + }, + "ranking": [ + { + "confidence": 1.0, + "intent_response_key": "chitchat/ask_weather", + }, + {"confidence": 0.0, "intent_response_key": "chitchat/ask_name"}, + ], + }, + }, + } + + default_processor._update_full_retrieval_intent(parse_data) + + assert parse_data[INTENT][INTENT_NAME_KEY] == "chitchat" + # assert that parse_data["intent"] has a key called response + assert FULL_RETRIEVAL_INTENT_NAME_KEY in parse_data[INTENT] + assert parse_data[INTENT][FULL_RETRIEVAL_INTENT_NAME_KEY] == "chitchat/ask_weather" diff --git a/tests/core/test_run.py b/tests/core/test_run.py index 1ac276d43772..8eda15058c0d 100644 --- a/tests/core/test_run.py +++ b/tests/core/test_run.py @@ -1,7 +1,6 @@ import warnings from unittest.mock import Mock -import aiohttp import pytest from typing import Text @@ -84,8 +83,6 @@ async def test_close_resources(loop: AbstractEventLoop): broker = SQLEventBroker() app = Mock() app.ctx.agent.tracker_store.event_broker = broker - app.ctx.agent.action_endpoint.session = aiohttp.ClientSession() - app.ctx.agent.model_server.session = aiohttp.ClientSession() with warnings.catch_warnings() as record: await run.close_resources(app, loop) diff --git a/tests/core/test_tracker_stores.py b/tests/core/test_tracker_stores.py index fb4a891b097e..4a05f370e5ff 100644 --- a/tests/core/test_tracker_stores.py +++ b/tests/core/test_tracker_stores.py @@ -614,6 +614,36 @@ async def test_sql_additional_events_with_session_start(domain: Domain): assert isinstance(additional_events[0], UserUttered) +async def test_tracker_store_retrieve_ordered_by_id( + domain: Domain, +): + tracker_store_kwargs = {"host": "sqlite:///"} + tracker_store = SQLTrackerStore(domain, **tracker_store_kwargs) + events = [ + SessionStarted(timestamp=1), + UserUttered("Hola", {"name": "greet"}, timestamp=2), + BotUttered("Hi", timestamp=2), + UserUttered("How are you?", {"name": "greet"}, timestamp=2), + BotUttered("I am good, whats up", timestamp=2), + UserUttered("Ciao", {"name": "greet"}, timestamp=2), + BotUttered("Bye", timestamp=2), + ] + sender_id = "test_sql_tracker_store_events_order" + tracker = DialogueStateTracker.from_events(sender_id, events) + await tracker_store.save(tracker) + + # Save other tracker to ensure that we don't run into problems with other senders + other_tracker = DialogueStateTracker.from_events("other-sender", [SessionStarted()]) + await tracker_store.save(other_tracker) + + # Retrieve tracker with events since latest SessionStarted + tracker = await tracker_store.retrieve(sender_id) + + assert len(tracker.events) == 7 + # assert the order of events is same as the order in which they were added + assert all((event == tracker.events[i] for i, event in enumerate(events))) + + @pytest.mark.parametrize( "tracker_store_type,tracker_store_kwargs", [(MockedMongoTrackerStore, {}), (SQLTrackerStore, {"host": "sqlite:///"})], diff --git a/tests/engine/storage/test_local_model_storage.py b/tests/engine/storage/test_local_model_storage.py index 9324c399c948..789aa0638568 100644 --- a/tests/engine/storage/test_local_model_storage.py +++ b/tests/engine/storage/test_local_model_storage.py @@ -2,21 +2,21 @@ import uuid from datetime import datetime from pathlib import Path -from tarsafe import TarSafe import freezegun import pytest from _pytest.monkeypatch import MonkeyPatch from _pytest.tmpdir import TempPathFactory +from tarsafe import TarSafe import rasa.shared.utils.io -from rasa.engine.graph import SchemaNode, GraphSchema, GraphModelConfiguration +from rasa.engine.graph import GraphModelConfiguration, GraphSchema, SchemaNode from rasa.engine.storage.local_model_storage import ( - LocalModelStorage, MODEL_ARCHIVE_METADATA_FILE, + LocalModelStorage, ) -from rasa.engine.storage.storage import ModelStorage, ModelMetadata from rasa.engine.storage.resource import Resource +from rasa.engine.storage.storage import ModelMetadata, ModelStorage from rasa.exceptions import UnsupportedModelVersionError from rasa.shared.core.domain import Domain from rasa.shared.data import TrainingType @@ -96,7 +96,7 @@ def test_read_long_resource_names_windows( domain: Domain, ): model_dir = tmp_path_factory.mktemp("model_dir") - version = "3.5.0" + version = "3.6.21" # full path length > 260 chars # but each component of the path needs to be below 255 chars diff --git a/tests/integration_tests/core/brokers/test_kafka.py b/tests/integration_tests/core/brokers/test_kafka.py index 89fcdbb2d7bd..6be6eaa89d48 100644 --- a/tests/integration_tests/core/brokers/test_kafka.py +++ b/tests/integration_tests/core/brokers/test_kafka.py @@ -1,9 +1,12 @@ +import pytest + from rasa.core.brokers.kafka import KafkaEventBroker from pytest import LogCaptureFixture import logging.config -def test_kafka_event_broker_valid(): +@pytest.mark.broker +async def test_kafka_event_broker_valid(): broker = KafkaEventBroker( url="localhost", topic="rasa", @@ -19,11 +22,11 @@ def test_kafka_event_broker_valid(): ) assert broker.producer.poll() == 1 finally: - broker.producer.flush() - broker._close() + await broker.close() -def test_kafka_event_broker_buffer_error_is_handled(caplog: LogCaptureFixture): +@pytest.mark.broker +async def test_kafka_event_broker_buffer_error_is_handled(caplog: LogCaptureFixture): broker = KafkaEventBroker( url="localhost", topic="rasa", @@ -48,5 +51,4 @@ def test_kafka_event_broker_buffer_error_is_handled(caplog: LogCaptureFixture): assert "Queue full" in caplog.text assert broker.producer.poll() == 1 finally: - broker.producer.flush() - broker._close() + await broker.close() diff --git a/tests/integration_tests/core/brokers/test_pika.py b/tests/integration_tests/core/brokers/test_pika.py index eb27f9ba9f09..b514b1f91c09 100644 --- a/tests/integration_tests/core/brokers/test_pika.py +++ b/tests/integration_tests/core/brokers/test_pika.py @@ -16,6 +16,7 @@ ) +@pytest.mark.broker async def test_pika_event_broker_connect(): broker = PikaEventBroker( host=RABBITMQ_HOST, @@ -31,6 +32,7 @@ async def test_pika_event_broker_connect(): await broker.close() +@pytest.mark.broker @pytest.mark.xdist_group("rabbitmq") async def test_pika_event_broker_publish_after_restart( docker_client: docker.DockerClient, @@ -102,6 +104,7 @@ async def test_pika_event_broker_publish_after_restart( rabbitmq_container.remove() +@pytest.mark.broker @pytest.mark.xdist_group("rabbitmq") @pytest.mark.parametrize("host_component", ["localhost", "myuser:mypassword@localhost"]) async def test_pika_event_broker_connect_with_path_and_query_params_in_url( diff --git a/tests/integration_tests/core/test_exporter.py b/tests/integration_tests/core/test_exporter.py new file mode 100644 index 000000000000..2b3e8b83edb4 --- /dev/null +++ b/tests/integration_tests/core/test_exporter.py @@ -0,0 +1,117 @@ +import textwrap +from pathlib import Path +from unittest.mock import Mock + +import pytest + +from pytest import MonkeyPatch + +from rasa.core.brokers.kafka import KafkaEventBroker +from rasa.core.exporter import Exporter +from rasa.core.tracker_store import InMemoryTrackerStore +from rasa.shared.core.domain import Domain +from rasa.shared.core.events import ActionExecuted +from rasa.shared.core.trackers import DialogueStateTracker + + +@pytest.mark.broker +async def test_exporter_publishes_to_kafka_broker_success( + tmp_path: Path, +) -> None: + tracker_store = InMemoryTrackerStore(domain=Domain.empty()) + tracker = DialogueStateTracker.from_events( + "test_export", + [ + ActionExecuted("action_listen"), + ], + ) + + await tracker_store.save(tracker) + + kafka_broker = KafkaEventBroker( + url="localhost", + topic="rasa", + sasl_username="admin", + sasl_password="password", + partition_by_sender=True, + ) + + endpoints_file = tmp_path / "endpoints.yml" + endpoints_file.write_text( + textwrap.dedent( + """ + event_broker: + type: kafka + topic: rasa + url: localhost:9092 + client_id: kafka-python-rasa + partition_by_sender: true + security_protocol: SASL_PLAINTEXT + sasl_username: admin + sasl_password: password + sasl_mechanism: PLAIN + """ + ) + ) + + exporter = Exporter(tracker_store, kafka_broker, str(endpoints_file)) + + published_events = await exporter.publish_events() + assert published_events == 1 + + +@pytest.mark.broker +async def test_exporter_publishes_to_kafka_broker_fail( + tmp_path: Path, + monkeypatch: MonkeyPatch, +) -> None: + tracker_store = InMemoryTrackerStore(domain=Domain.empty()) + tracker = DialogueStateTracker.from_events( + "test_export", + [ + ActionExecuted("action_listen"), + ], + ) + + await tracker_store.save(tracker) + + kafka_broker = KafkaEventBroker( + url="localhost", + topic="rasa", + sasl_username="admin", + sasl_password="password", + partition_by_sender=True, + ) + + endpoints_file = tmp_path / "endpoints.yml" + endpoints_file.write_text( + textwrap.dedent( + """ + event_broker: + type: kafka + topic: rasa + url: localhost:9092 + client_id: kafka-python-rasa + partition_by_sender: true + security_protocol: SASL_PLAINTEXT + sasl_username: admin + sasl_password: password + sasl_mechanism: PLAIN + """ + ) + ) + + exporter = Exporter(tracker_store, kafka_broker, str(endpoints_file)) + + # patch the exporter to raise an exception when publishing events + monkeypatch.setattr(exporter, "publish_events", Mock(side_effect=Exception)) + + with pytest.raises(Exception) as error: + await exporter.publish_events() + assert "Producer terminating with 1 messages" in str(error.value) + assert ( + "still in queue or transit: use flush() to wait for " + "outstanding message delivery" in str(error.value) + ) + # necessary for producer teardown + await kafka_broker.close() diff --git a/tests/nlu/classifiers/test_diet_classifier.py b/tests/nlu/classifiers/test_diet_classifier.py index 1f0c37a85faa..1fd84fdac47d 100644 --- a/tests/nlu/classifiers/test_diet_classifier.py +++ b/tests/nlu/classifiers/test_diet_classifier.py @@ -971,24 +971,35 @@ async def test_no_bilou_when_entity_recognition_off( @pytest.mark.timeout(120, func_only=True) @pytest.mark.parametrize( - "batch_size, expected_num_batches", + "batch_size, expected_num_batches, drop_small_last_batch", # the training dataset has 48 NLU examples [ - (1, 48), - (8, 6), - (15, 3), - (16, 3), - (18, 3), - (20, 2), - (32, 2), - (64, 1), - (128, 1), - (256, 1), + (1, 48, True), + (8, 6, True), + (15, 3, True), + (16, 3, True), + (18, 3, True), + (20, 2, True), + (32, 2, True), + (64, 1, True), + (128, 1, True), + (256, 1, True), + (1, 48, False), + (8, 6, False), + (15, 4, False), + (16, 3, False), + (18, 3, False), + (20, 3, False), + (32, 2, False), + (64, 1, False), + (128, 1, False), + (256, 1, False), ], ) async def test_dropping_of_last_partial_batch( batch_size: int, expected_num_batches: int, + drop_small_last_batch: bool, create_diet: Callable[..., DIETClassifier], train_and_preprocess: Callable[..., Tuple[TrainingData, List[GraphComponent]]], ): @@ -1012,7 +1023,9 @@ async def test_dropping_of_last_partial_batch( ) model_data = diet.preprocess_train_data(training_data) - data_generator, _ = train_utils.create_data_generators(model_data, batch_size, 1) + data_generator, _ = train_utils.create_data_generators( + model_data, batch_size, 1, drop_small_last_batch=drop_small_last_batch + ) assert len(data_generator) == expected_num_batches @@ -1041,6 +1054,8 @@ async def test_dropping_of_last_partial_batch_empty_data( ) model_data = diet.preprocess_train_data(training_data) - data_generator, _ = train_utils.create_data_generators(model_data, 64, 1) + data_generator, _ = train_utils.create_data_generators( + model_data, 64, 1, drop_small_last_batch=True + ) assert len(data_generator) == 0 diff --git a/tests/nlu/extractors/test_crf_entity_extractor.py b/tests/nlu/extractors/test_crf_entity_extractor.py index b2342ab30fb7..bf095385017b 100644 --- a/tests/nlu/extractors/test_crf_entity_extractor.py +++ b/tests/nlu/extractors/test_crf_entity_extractor.py @@ -1,23 +1,25 @@ import copy from typing import Dict, Text, List, Any, Callable +import numpy as np import pytest from rasa.engine.graph import ExecutionContext from rasa.engine.storage.resource import Resource from rasa.engine.storage.storage import ModelStorage +from rasa.nlu.constants import SPACY_DOCS +from rasa.nlu.extractors.crf_entity_extractor import ( + CRFEntityExtractor, + CRFEntityExtractorOptions, + CRFToken, +) from rasa.nlu.featurizers.dense_featurizer.spacy_featurizer import SpacyFeaturizer from rasa.nlu.tokenizers.spacy_tokenizer import SpacyTokenizer -from rasa.nlu.constants import SPACY_DOCS from rasa.nlu.tokenizers.whitespace_tokenizer import WhitespaceTokenizer from rasa.nlu.utils.spacy_utils import SpacyModel, SpacyNLP from rasa.shared.importers.rasa import RasaFileImporter from rasa.shared.nlu.constants import TEXT, ENTITIES from rasa.shared.nlu.training_data.message import Message -from rasa.nlu.extractors.crf_entity_extractor import ( - CRFEntityExtractor, - CRFEntityExtractorOptions, -) @pytest.fixture() @@ -204,7 +206,7 @@ def test_crf_use_dense_features( spacy_featurizer.process([message]) text_data = crf_extractor._convert_to_crf_tokens(message) - features = crf_extractor._crf_tokens_to_features(text_data) + features = crf_extractor._crf_tokens_to_features(text_data, component_config) assert "0:text_dense_features" in features[0] dense_features, _ = message.get_dense_features(TEXT, []) @@ -249,3 +251,110 @@ def test_process_unfeaturized_input( assert processed_message.get(TEXT) == message_text assert processed_message.get(ENTITIES) == [] + + +@pytest.fixture +def sample_data(): + return { + "text": "apple", + "pos_tag": "NOUN", + "pattern": {"length": 5, "is_capitalized": False}, + "dense_features": np.array([0.1, 0.2, 0.3]), + "entity_tag": "B-FOOD", + "entity_role_tag": "INGREDIENT", + "entity_group_tag": "ITEM", + } + + +@pytest.fixture +def sample_token(sample_data): + return CRFToken( + sample_data["text"], + sample_data["pos_tag"], + sample_data["pattern"], + sample_data["dense_features"], + sample_data["entity_tag"], + sample_data["entity_role_tag"], + sample_data["entity_group_tag"], + ) + + +def test_crf_token_to_dict(sample_data, sample_token): + token_dict = sample_token.to_dict() + + assert token_dict["text"] == sample_data["text"] + assert token_dict["pos_tag"] == sample_data["pos_tag"] + assert token_dict["pattern"] == sample_data["pattern"] + assert token_dict["dense_features"] == [ + str(x) for x in sample_data["dense_features"] + ] + assert token_dict["entity_tag"] == sample_data["entity_tag"] + assert token_dict["entity_role_tag"] == sample_data["entity_role_tag"] + assert token_dict["entity_group_tag"] == sample_data["entity_group_tag"] + + +def test_crf_token_create_from_dict(sample_data): + dict_data = { + "text": sample_data["text"], + "pos_tag": sample_data["pos_tag"], + "pattern": sample_data["pattern"], + "dense_features": [str(x) for x in sample_data["dense_features"]], + "entity_tag": sample_data["entity_tag"], + "entity_role_tag": sample_data["entity_role_tag"], + "entity_group_tag": sample_data["entity_group_tag"], + } + + token = CRFToken.create_from_dict(dict_data) + + assert token.text == sample_data["text"] + assert token.pos_tag == sample_data["pos_tag"] + assert token.pattern == sample_data["pattern"] + np.testing.assert_array_equal(token.dense_features, sample_data["dense_features"]) + assert token.entity_tag == sample_data["entity_tag"] + assert token.entity_role_tag == sample_data["entity_role_tag"] + assert token.entity_group_tag == sample_data["entity_group_tag"] + + +def test_crf_token_roundtrip_conversion(sample_token): + token_dict = sample_token.to_dict() + new_token = CRFToken.create_from_dict(token_dict) + + assert new_token.text == sample_token.text + assert new_token.pos_tag == sample_token.pos_tag + assert new_token.pattern == sample_token.pattern + np.testing.assert_array_equal(new_token.dense_features, sample_token.dense_features) + assert new_token.entity_tag == sample_token.entity_tag + assert new_token.entity_role_tag == sample_token.entity_role_tag + assert new_token.entity_group_tag == sample_token.entity_group_tag + + +def test_crf_token_empty_dense_features(sample_data): + sample_data["dense_features"] = np.array([]) + token = CRFToken( + sample_data["text"], + sample_data["pos_tag"], + sample_data["pattern"], + sample_data["dense_features"], + sample_data["entity_tag"], + sample_data["entity_role_tag"], + sample_data["entity_group_tag"], + ) + token_dict = token.to_dict() + new_token = CRFToken.create_from_dict(token_dict) + np.testing.assert_array_equal(new_token.dense_features, np.array([])) + + +def test_crf_token_empty_pattern(sample_data): + sample_data["pattern"] = {} + token = CRFToken( + sample_data["text"], + sample_data["pos_tag"], + sample_data["pattern"], + sample_data["dense_features"], + sample_data["entity_tag"], + sample_data["entity_role_tag"], + sample_data["entity_group_tag"], + ) + token_dict = token.to_dict() + new_token = CRFToken.create_from_dict(token_dict) + assert new_token.pattern == {} diff --git a/tests/shared/nlu/training_data/test_features.py b/tests/shared/nlu/training_data/test_features.py index bd0f29fa046b..457e9648f28f 100644 --- a/tests/shared/nlu/training_data/test_features.py +++ b/tests/shared/nlu/training_data/test_features.py @@ -1,17 +1,56 @@ import itertools +import os +import tempfile +from pathlib import Path from typing import Optional, Text, List, Dict, Tuple, Any import numpy as np import pytest import scipy.sparse -from rasa.shared.nlu.training_data.features import Features from rasa.shared.nlu.constants import ( FEATURE_TYPE_SENTENCE, FEATURE_TYPE_SEQUENCE, TEXT, INTENT, ) +from rasa.shared.nlu.training_data.features import ( + Features, + FeatureMetadata, + save_features, + load_features, +) + + +@pytest.fixture +def safe_tensors_tmp_file() -> str: + with tempfile.NamedTemporaryFile(delete=False, suffix=".safetensors") as f: + yield f.name + os.unlink(f.name) + + +@pytest.fixture +def dense_features() -> Features: + features_matrix = np.array([[1, 2, 3], [4, 5, 6]]) + return Features( + features=features_matrix, + feature_type="dense", + attribute="test", + origin="test_origin", + ) + + +@pytest.fixture +def sparse_features() -> Features: + features_matrix = scipy.sparse.csr_matrix( + ([1, 2, 3], ([0, 1, 1], [0, 1, 2])), shape=(2, 3) + ) + return Features( + features=features_matrix, + feature_type="sparse", + attribute="test", + origin="test_origin", + ) @pytest.mark.parametrize( @@ -181,6 +220,7 @@ def _generate_feature_list_and_modifications( instantiate `Features` that differ from the aforementioned list of features in exactly one property (i.e. type, sequence length (if the given `type` is sequence type only), attribute, origin) + Args: is_sparse: whether all features should be sparse type: the type to be used for all features @@ -190,7 +230,6 @@ def _generate_feature_list_and_modifications( a list of kwargs dictionaries that can be used to instantiate `Features` that differ from the aforementioned list of features in exactly one property """ - seq_len = 3 first_dim = 1 if type == FEATURE_TYPE_SENTENCE else 3 @@ -467,3 +506,179 @@ def test_reduce_raises_if_combining_different_origins_or_attributes(differ: Text expected_origin = ["origin-1"] with pytest.raises(ValueError, match=message): Features.reduce(features_list, expected_origins=expected_origin) + + +def test_feature_metadata(): + metadata = FeatureMetadata( + data_type="dense", + attribute="text", + origin="test", + is_sparse=False, + shape=(10, 5), + safetensors_key="key_0", + ) + + assert metadata.data_type == "dense" + assert metadata.attribute == "text" + assert metadata.origin == "test" + assert not metadata.is_sparse + assert metadata.shape == (10, 5) + assert metadata.safetensors_key == "key_0" + + +def test_save_dense_features(safe_tensors_tmp_file: str, dense_features: Features): + features_dict = {"test_key": [dense_features]} + metadata = save_features(features_dict, safe_tensors_tmp_file) + + assert "test_key" in metadata + assert len(metadata["test_key"]) == 1 + assert metadata["test_key"][0]["data_type"] == "dense" + assert metadata["test_key"][0]["shape"] == (2, 3) + assert not metadata["test_key"][0]["is_sparse"] + assert Path(safe_tensors_tmp_file).exists() + + +def test_save_sparse_features(safe_tensors_tmp_file: str, sparse_features: Features): + features_dict = {"test_key": [sparse_features]} + metadata = save_features(features_dict, safe_tensors_tmp_file) + + assert "test_key" in metadata + assert len(metadata["test_key"]) == 1 + assert metadata["test_key"][0]["data_type"] == "sparse" + assert metadata["test_key"][0]["shape"] == (2, 3) + assert metadata["test_key"][0]["is_sparse"] + assert Path(safe_tensors_tmp_file).exists() + + +def test_save_mixed_features( + safe_tensors_tmp_file: str, dense_features: Features, sparse_features: Features +): + features_dict = {"test_key": [dense_features, sparse_features]} + metadata = save_features(features_dict, safe_tensors_tmp_file) + + assert "test_key" in metadata + assert len(metadata["test_key"]) == 2 + assert metadata["test_key"][0]["data_type"] == "dense" + assert metadata["test_key"][1]["data_type"] == "sparse" + assert Path(safe_tensors_tmp_file).exists() + + +def test_save_multiple_keys( + safe_tensors_tmp_file: str, dense_features: Features, sparse_features: Features +): + features_dict = {"dense_key": [dense_features], "sparse_key": [sparse_features]} + metadata = save_features(features_dict, safe_tensors_tmp_file) + + assert "dense_key" in metadata + assert "sparse_key" in metadata + assert metadata["dense_key"][0]["data_type"] == "dense" + assert metadata["sparse_key"][0]["data_type"] == "sparse" + assert Path(safe_tensors_tmp_file).exists() + + +@pytest.fixture +def setup_save_load( + safe_tensors_tmp_file: str, dense_features: Features, sparse_features: Features +) -> Tuple[str, Dict[str, Any], Dict[str, List[Features]]]: + features_dict = {"dense_key": [dense_features], "sparse_key": [sparse_features]} + metadata = save_features(features_dict, safe_tensors_tmp_file) + return safe_tensors_tmp_file, metadata, features_dict + + +def test_load_dense_features( + setup_save_load: Tuple[str, Dict[str, Any], Dict[str, List[Features]]], +): + temp_file, metadata, original_dict = setup_save_load + loaded_dict = load_features(temp_file, metadata) + + assert "dense_key" in loaded_dict + assert len(loaded_dict["dense_key"]) == 1 + assert not loaded_dict["dense_key"][0].is_sparse() + np.testing.assert_array_equal( + loaded_dict["dense_key"][0].features, original_dict["dense_key"][0].features + ) + + +def test_load_sparse_features( + setup_save_load: Tuple[str, Dict[str, Any], Dict[str, List[Features]]], +): + temp_file, metadata, original_dict = setup_save_load + loaded_dict = load_features(temp_file, metadata) + + assert "sparse_key" in loaded_dict + assert len(loaded_dict["sparse_key"]) == 1 + assert loaded_dict["sparse_key"][0].is_sparse() + assert ( + loaded_dict["sparse_key"][0].features != original_dict["sparse_key"][0].features + ).nnz == 0 + + +def test_load_preserves_metadata( + setup_save_load: Tuple[str, Dict[str, Any], Dict[str, List[Features]]], +): + temp_file, metadata, original_dict = setup_save_load + loaded_dict = load_features(temp_file, metadata) + + for key in original_dict: + for orig_feat, loaded_feat in zip(original_dict[key], loaded_dict[key]): + assert orig_feat.type == loaded_feat.type + assert orig_feat.attribute == loaded_feat.attribute + assert orig_feat.origin == loaded_feat.origin + + +def test_load_nonexistent_file(): + with pytest.raises(Exception): + load_features("nonexistent.safetensors", {}) + + +def test_load_invalid_metadata(safe_tensors_tmp_file: str, dense_features: Features): + features_dict = {"test_key": [dense_features]} + metadata = save_features(features_dict, safe_tensors_tmp_file) + + # Corrupt the metadata + metadata["test_key"][0]["safetensors_key"] = "invalid_key" + + with pytest.raises(Exception): + load_features(safe_tensors_tmp_file, metadata) + + +def test_end_to_end(safe_tensors_tmp_file: str): + # Create test data + dense_matrix = np.array([[1, 2], [3, 4]]) + sparse_matrix = scipy.sparse.csr_matrix(([1, 2], ([0, 1], [0, 1])), shape=(2, 2)) + + features_dict = { + "group1": [ + Features(dense_matrix, "dense", "test1", "origin1"), + Features(sparse_matrix, "sparse", "test2", "origin2"), + ], + "group2": [ + Features(dense_matrix * 2, "dense", "test3", ["origin3", "origin4"]) + ], + } + + # Save features + metadata = save_features(features_dict, safe_tensors_tmp_file) + + # Load features + loaded_dict = load_features(safe_tensors_tmp_file, metadata) + + # Verify structure + assert set(loaded_dict.keys()) == set(features_dict.keys()) + assert len(loaded_dict["group1"]) == 2 + assert len(loaded_dict["group2"]) == 1 + + # Verify dense features + np.testing.assert_array_equal( + loaded_dict["group1"][0].features, features_dict["group1"][0].features + ) + + # Verify sparse features + assert ( + loaded_dict["group1"][1].features != features_dict["group1"][1].features + ).nnz == 0 + + # Verify metadata + assert loaded_dict["group1"][0].type == "dense" + assert loaded_dict["group1"][1].type == "sparse" + assert loaded_dict["group2"][0].origin == ["origin3", "origin4"] diff --git a/tests/utils/tensorflow/test_feature_array.py b/tests/utils/tensorflow/test_feature_array.py new file mode 100644 index 000000000000..95be7ba993a6 --- /dev/null +++ b/tests/utils/tensorflow/test_feature_array.py @@ -0,0 +1,197 @@ +import numpy as np +import scipy.sparse + +from rasa.utils.tensorflow.feature_array import ( + _recursive_serialize, + _serialize_nested_data, + _deserialize_nested_data, +) +from rasa.utils.tensorflow.model_data import RasaModelData + + +def test_recursive_serialize_numpy_array(): + data_dict = {} + metadata = [] + + _recursive_serialize(np.array([1, 2, 3]), "test_array", data_dict, metadata) + assert "test_array_array" in data_dict + assert metadata[0] == {"type": "dense", "key": "test_array_array", "shape": (3,)} + + +def test_recursive_serialize_floats(): + data_dict = {} + metadata = [] + + _recursive_serialize([1.0, 2.0, 3.0], "test_list", data_dict, metadata) + assert "test_list_list" in data_dict + assert metadata[0] == {"type": "list", "key": "test_list_list"} + + +def test_recursive_serialize_sparse_matrix(): + data_dict = {} + metadata = [] + + sparse_matrix = scipy.sparse.random(5, 10, density=0.1, format="coo") + _recursive_serialize(sparse_matrix, "test_sparse", data_dict, metadata) + assert "test_sparse_data" in data_dict + assert "test_sparse_row" in data_dict + assert "test_sparse_col" in data_dict + assert metadata[0] == { + "type": "sparse", + "key": "test_sparse", + "shape": sparse_matrix.shape, + } + + +def test_serialize_model_data(model_data: RasaModelData): + nested_data = model_data.data + + data_dict = {} + metadata = [] + _serialize_nested_data(nested_data, "component", data_dict, metadata) + + assert len(metadata) == 5 + + assert metadata[0]["key"] == "text" + assert len(metadata[0]["components"]) == 1 + assert metadata[0]["components"][0]["key"] == "sentence" + assert metadata[0]["components"][0]["number_of_dimensions"] == 3 + assert len(metadata[0]["components"][0]["features"]) == 2 + assert metadata[0]["components"][0]["features"][0]["type"] == "group" + assert len(metadata[0]["components"][0]["features"][0]["subcomponents"]) == 5 + assert ( + metadata[0]["components"][0]["features"][0]["subcomponents"][0]["type"] + == "dense" + ) + assert metadata[0]["components"][0]["features"][0]["subcomponents"][0]["shape"] == ( + 5, + 14, + ) + assert metadata[0]["components"][0]["features"][1]["type"] == "group" + assert len(metadata[0]["components"][0]["features"][1]["subcomponents"]) == 5 + assert ( + metadata[0]["components"][0]["features"][1]["subcomponents"][0]["type"] + == "sparse" + ) + assert metadata[0]["components"][0]["features"][1]["subcomponents"][0]["shape"] == ( + 5, + 10, + ) + + assert metadata[3]["key"] == "label" + assert len(metadata[3]["components"]) == 1 + assert metadata[3]["components"][0]["key"] == "ids" + assert metadata[3]["components"][0]["number_of_dimensions"] == 1 + assert metadata[3]["components"][0]["features"][0]["type"] == "list" + assert ( + metadata[3]["components"][0]["features"][0]["key"] + == "component_label_ids_0_list" + ) + + assert len(data_dict) == 87 + assert ( + data_dict["component_label_ids_0_list"] + == model_data.data["label"]["ids"][0].view(np.ndarray) + ).all() + + +def test_serialize_and_deserialize_model_data(model_data: RasaModelData): + actual_data = model_data.data + + data_dict = {} + metadata = [] + _serialize_nested_data(actual_data, "component", data_dict, metadata) + + loaded_data = _deserialize_nested_data(metadata, data_dict) + + assert len(actual_data) == len(loaded_data) + + assert len(actual_data["text"]["sentence"]) == len(loaded_data["text"]["sentence"]) + + # text.sentence has a dimension of 3 + assert len(actual_data["text"]["sentence"][0]) == len( + loaded_data["text"]["sentence"][0] + ) + # assert that the numpy arrays of the actual and loaded data in + # text.sentence are the same + for i in range(0, 5): + assert ( + actual_data["text"]["sentence"][0][i] + == loaded_data["text"]["sentence"][0][i] + ).all() + assert len(actual_data["text"]["sentence"][1]) == len( + loaded_data["text"]["sentence"][1] + ) + # assert that the sparse matrices of the actual and loaded data in + # text.sentence are the same + for i in range(0, 5): + assert ( + actual_data["text"]["sentence"][1][i] + == loaded_data["text"]["sentence"][1][i] + ).data.all() + + # action_text.sequence has a dimension of 4 + assert len(actual_data["action_text"]["sequence"]) == len( + loaded_data["action_text"]["sequence"] + ) + assert len(actual_data["action_text"]["sequence"][0]) == len( + loaded_data["action_text"]["sequence"][0] + ) + # assert that the sparse matrices of the actual and loaded data in + # action_text.sequence are the same + for i in range(0, 5): + for j in range(0, len(actual_data["action_text"]["sequence"][0][i])): + assert ( + actual_data["action_text"]["sequence"][0][i][j] + == loaded_data["action_text"]["sequence"][0][i][j] + ).data.all() + assert len(actual_data["action_text"]["sequence"][1]) == len( + loaded_data["action_text"]["sequence"][1] + ) + # assert that the numpy array of the actual and loaded data in + # action_text.sequence are the same + for i in range(0, 5): + for j in range(0, len(actual_data["action_text"]["sequence"][1][i])): + assert ( + actual_data["action_text"]["sequence"][1][i][j] + == loaded_data["action_text"]["sequence"][1][i][j] + ).all() + + # dialogue.sentence has a dimension of 3 + assert len(actual_data["dialogue"]["sentence"]) == len( + loaded_data["dialogue"]["sentence"] + ) + assert len(actual_data["dialogue"]["sentence"][0]) == len( + loaded_data["dialogue"]["sentence"][0] + ) + # assert that the numpy array of the actual and loaded data in + # dialogue.sentence are the same + for i in range(0, 5): + assert ( + actual_data["dialogue"]["sentence"][0][i] + == loaded_data["dialogue"]["sentence"][0][i] + ).all() + + # label.ids has a dimension of 4 + assert len(actual_data["label"]["ids"]) == len(loaded_data["label"]["ids"]) + # assert that the numpy array of the actual and loaded data in + # label.ids are the same + assert ( + actual_data["label"]["ids"][0].view(np.ndarray) + == loaded_data["label"]["ids"][0].view(np.ndarray) + ).all() + + # entities.tag_ids has a dimension of 3 + assert len(actual_data["entities"]["tag_ids"]) == len( + loaded_data["entities"]["tag_ids"] + ) + assert len(actual_data["entities"]["tag_ids"][0]) == len( + loaded_data["entities"]["tag_ids"][0] + ) + # assert that the numpy array of the actual and loaded data in + # entities.tag_ids are the same + for i in range(0, 5): + assert ( + actual_data["entities"]["tag_ids"][0][i] + == loaded_data["entities"]["tag_ids"][0][i] + ).all() diff --git a/tests/utils/test_endpoints.py b/tests/utils/test_endpoints.py index 071e54ee9318..711f2fd25faa 100644 --- a/tests/utils/test_endpoints.py +++ b/tests/utils/test_endpoints.py @@ -1,4 +1,4 @@ -import logging +import structlog from pathlib import Path from typing import Text, Optional, Union from unittest.mock import Mock @@ -35,13 +35,14 @@ def test_concat_url(base, subpath, expected_result): assert endpoint_utils.concat_url(base, subpath) == expected_result -def test_warning_for_base_paths_with_trailing_slash(caplog): +def test_warning_for_base_paths_with_trailing_slash(): test_path = "base/" - - with caplog.at_level(logging.DEBUG, logger="rasa.utils.endpoints"): + with structlog.testing.capture_logs() as caplog: assert endpoint_utils.concat_url(test_path, None) == test_path - assert len(caplog.records) == 1 + assert len(caplog) == 1 + assert caplog[0]["event"] == "endpoint.concat_url.trailing_slash" + assert caplog[0]["log_level"] == "debug" async def test_endpoint_config(): @@ -88,7 +89,7 @@ async def test_endpoint_config(): # unfortunately, the mock library won't report any headers stored on # the session object, so we need to verify them separately - async with endpoint.session as s: + async with endpoint.session() as s: assert s._default_headers.get("X-Powered-By") == "Rasa" assert s._default_auth.login == "user" assert s._default_auth.password == "pass" @@ -231,32 +232,3 @@ def test_int_arg(value: Optional[Union[int, str]], default: int, expected_result if value is not None: request.args = {"key": value} assert endpoint_utils.int_arg(request, "key", default) == expected_result - - -async def test_endpoint_config_caches_session() -> None: - """Test that the EndpointConfig session is cached. - - Assert identity of the session object, which should not be recreated when calling - the property `session` multiple times. - """ - endpoint = endpoint_utils.EndpointConfig("https://example.com/") - session = endpoint.session - - assert endpoint.session is session - - # teardown - await endpoint.session.close() - - -async def test_endpoint_config_constructor_does_not_create_session_cached_property() -> None: # noqa: E501 - """Test that the instantiation of EndpointConfig does not create the session cached property.""" # noqa: E501 - endpoint = endpoint_utils.EndpointConfig("https://example.com/") - - assert endpoint.__dict__.get("url") == "https://example.com/" - assert endpoint.__dict__.get("session") is None - - # the property is created when it is accessed - async with endpoint.session as session: - assert session is not None - - assert endpoint.__dict__.get("session") is session diff --git a/tests/utils/test_io.py b/tests/utils/test_io.py index ac788373a422..3042b113aaf3 100644 --- a/tests/utils/test_io.py +++ b/tests/utils/test_io.py @@ -1,5 +1,3 @@ -from pathlib import Path -from typing import Dict, Text import pytest from _pytest.tmpdir import TempPathFactory from prompt_toolkit.document import Document @@ -71,22 +69,6 @@ def is_valid(user_input) -> None: assert e.value.message == error_message -@pytest.mark.parametrize( - "input,kwargs,expected", - [ - ({(1, 2): 3}, {}, {repr((1, 2)): 3}), - ({(1, 2): 3}, {"encode_non_string_keys": True}, {(1, 2): 3}), - ], -) -def test_write_and_load_dict_via_jsonpickle( - tmp_path: Path, input: Dict, kwargs: Dict[Text, bool], expected: Dict -): - file_name = tmp_path / "bla.pkl" - rasa.utils.io.json_pickle(file_name=file_name, obj=input, **kwargs) - loaded = rasa.utils.io.json_unpickle(file_name=file_name, **kwargs) - assert loaded == expected - - def test_empty_directories_are_equal(tmp_path_factory: TempPathFactory): dir1 = tmp_path_factory.mktemp("dir1") dir2 = tmp_path_factory.mktemp("dir2")