diff --git a/.github/workflows/buildwheels.yml b/.github/workflows/buildwheels.yml deleted file mode 100644 index 10d6ee6c7..000000000 --- a/.github/workflows/buildwheels.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Build - -on: [push, pull_request] - -jobs: - build_wheels: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - #os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, windows-11-arm, macos-15-intel, macos-latest] - os: [ubuntu-latest] - - steps: - - uses: actions/checkout@v5 - - # Used to host cibuildwheel - - uses: actions/setup-python@v5 - - - name: Install cibuildwheel - run: python -m pip install cibuildwheel - - - name: Build wheels - run: python -m cibuildwheel --output-dir wheelhouse - env: - CIBW_BUILD: cp314-manylinux_x86_64 cp314-musllinux_x86_64 - CIBW_ENVIRONMENT: CINDERX_ENABLE_PGO=1 CINDERX_ENABLE_LTO=1 - CIBW_BUILD_VERBOSITY: 3 - - - uses: actions/upload-artifact@v4 - with: - name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} - path: ./wheelhouse/*.whl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12282c459..9b44386d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,40 +1,140 @@ -name: CinderX +name: CI -on: [push] +on: [push, pull_request] jobs: - test: - runs-on: ubuntu-latest + run_tests: + name: Run Tests + runs-on: ${{ matrix.os }} + strategy: + # Don't let test failures on one platform block other platforms. + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest] + # Test against multiple patch versions as CinderX references internal + # CPython functions and structures which can lead to ABI problems. See + # cinderx/UpstreamBorrow for more details. + python-version: ['3.14.3', '3.14.4', '3.14.5', '3.14.6'] + include: + - os: ubuntu-latest + python-version: '3.14t' + - os: ubuntu-latest + python-version: '3.15.0-beta.3' steps: - name: Checkout CinderX - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 with: - python-version: 3.14.3 + python-version: ${{ matrix.python-version }} + allow-prereleases: true - name: Set up uv uses: astral-sh/setup-uv@v7 + # Ubuntu's default GCC 13.3 hits an internal compiler error on + # SwapLockGuard's std::atomic_ref::compare_exchange_weak in + # cinderx/Jit/code_patcher.cpp, which is only compiled under + # Py_GIL_DISABLED. GCC 14 (which is also what manylinux_2_28 + # ships via gcc-toolset-14) compiles it cleanly. + - name: Install GCC 14 + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y gcc-14 g++-14 + echo "CC=gcc-14" >> "$GITHUB_ENV" + echo "CXX=g++-14" >> "$GITHUB_ENV" + - name: Create venv - run: uv venv --python 3.14.3 + run: uv venv --python ${{ matrix.python-version }} - name: Build CinderX - run: uv build --wheel --python 3.14.3 + # RuntimeTests do not support Windows or free-threaded builds yet. + env: + CINDERX_BUILD_RUNTIME_TESTS: ${{ runner.os != 'Windows' && matrix.python-version != '3.14t' && '1' || '' }} + CINDERX_RUNTIME_TESTS_OUTPUT_DIR: ${{ github.workspace }}/build/runtime-tests + run: uv build --wheel --python ${{ matrix.python-version }} - name: Install CinderX - run: uv pip install dist/*.whl + run: uv pip install cinderx --find-links dist --no-index - - name: Check CinderX Imports + - name: Check CinderX loads successfully run: | uv run python -c 'import cinderx ; print(cinderx.get_import_error()) ; assert cinderx.is_initialized()' - - name: Install Pytest - run: | - uv pip install pytest + - name: Install pytest + run: uv pip install pytest + + - name: Run Tests + # Free-threaded builds aren't stable yet, so don't run the test suite + # against them. + if: matrix.python-version != '3.14t' + run: uv run pytest cinderx/PythonLib/test_cinderx/ - - name: Run Basic JIT Tests + - name: Run native tests + if: runner.os != 'Windows' && matrix.python-version != '3.14t' + # Skip known OSS-only failures. Keep these narrow so RuntimeTests still + # provide useful signal. + # TODO(T275133043): burn these down as the underlying issues are fixed. + # Particularly the segfaults. run: | - uv run pytest cinderx/PythonLib/test_cinderx/test_jit_{async_generators,count_calls,disable,exception,frame,generators,global_cache,perf_map,specialization,type_annotations}.py + set -euo pipefail + common='AllPassesTest.*:AllPassesStaticTest.*:CmdLineTest.ExplicitJITDisable:UtilTest.SymbolizerResolvesDynamicSymbol' + arm64='LIRGeneratorTest.ParserTest:BackendTest.MoveSequenceOpt2Test:CodePatcherTest.DeoptPatch:BranchRelaxationTest.*' + macos='BackendTest.CastTest:BackendTest.InlineJITRTCastTest:SimplifyTest.BinaryOpWithObjSpecLeftAndRightFloatExactTurnsIntoLoadConst:NativeCallsTest.NativeInvokeBasic:SimplifyStaticTest.UnboxOfRandMaxIsEliminated:DeadCodeEliminationAndSimplifyTest.UnboxOfStaticGlobalIsOptimized:HIRBuilderStaticTest.CIntTypeEmitsConvertPrimitive:CodePatcherTest.DeoptPatch' + filter="-${common}" + if [[ "$RUNNER_ARCH" == "ARM64" ]]; then + filter="${filter}:${arm64}" + fi + if [[ "$RUNNER_OS" == "macOS" ]]; then + filter="${filter}:${macos}" + fi + package_parent="$(uv run python -c 'import pathlib, cinderx; print(pathlib.Path(cinderx.__file__).resolve().parent.parent)')" + export PYTHONPATH="$package_parent" + export GTEST_FILTER="$filter" + (cd cinderx && ../build/runtime-tests/RuntimeTests) + + build_wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + # Don't let build failures on one platform block other platforms. + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest] + + steps: + - name: Checkout CinderX + uses: actions/checkout@v6 + + # cibuildwheel uses Python from a docker container. The Python version is + # controlled by selecting the docker image version in pyproject.toml. + - name: Set up Python + uses: actions/setup-python@v6 + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel + + - name: Build wheels + run: python -m cibuildwheel --output-dir wheelhouse + + build_sdist: + name: Build source distribution + runs-on: ubuntu-latest + + steps: + - name: Checkout CinderX + uses: actions/checkout@v6 + + # An sdist is just source code and doesn't contain compiled artifacts, it + # doesn't need to be built per patch version. + - name: Set up Python + uses: actions/setup-python@v6 + + - name: Install build tools + run: python -m pip install build + + - name: Build sdist + run: python -m build --sdist diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000..e64c56edb --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,69 @@ +name: Documentation + +on: + # Deploy when docs or the website change on main. + push: + branches: [main] + paths: + - 'cinderx/website/**' + # Build-only check on PRs that touch docs. + pull_request: + paths: + - 'cinderx/website/**' + # Allow manual runs. + workflow_dispatch: + +# Allow the deploy job to publish to GitHub Pages via OIDC. +permissions: + contents: read + pages: write + id-token: write + +# Only one concurrent deploy; don't cancel an in-progress production deploy. +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + if: github.repository == 'facebookincubator/cinderx' + name: Build site + runs-on: ubuntu-latest + defaults: + run: + working-directory: cinderx/website + steps: + - name: Checkout CinderX + uses: actions/checkout@v6 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: yarn + cache-dependency-path: cinderx/website/yarn.lock + + - name: Install dependencies + run: yarn install + + - name: Build (fails on broken links) + run: yarn build + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: cinderx/website/build + + deploy: + # Only deploy from main; PRs run the build job above as a check. + if: github.repository == 'facebookincubator/cinderx' && github.event_name != 'pull_request' + name: Deploy to GitHub Pages + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/getdeps-3_14-linux.yml b/.github/workflows/getdeps-3_14-linux.yml deleted file mode 100644 index e34ba59d0..000000000 --- a/.github/workflows/getdeps-3_14-linux.yml +++ /dev/null @@ -1,166 +0,0 @@ -# This file was @generated by getdeps.py - -name: getdeps-3_14-Linux - -on: - push: - branches: - - main - pull_request: - branches: - - main - -permissions: - contents: read # to fetch code (actions/checkout) - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - id: paths - name: Query paths - run: python3 build/fbcode_builder/getdeps.py query-paths --recursive --src-dir=. cinderx-3_14 >> "$GITHUB_OUTPUT" - - name: Fetch ninja - if: ${{ steps.paths.outputs.ninja_SOURCE }} - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests ninja - - name: Fetch cmake - if: ${{ steps.paths.outputs.cmake_SOURCE }} - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests cmake - - name: Fetch python-setuptools - if: ${{ steps.paths.outputs.python-setuptools_SOURCE }} - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests python-setuptools - - name: Fetch autoconf - if: ${{ steps.paths.outputs.autoconf_SOURCE }} - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests autoconf - - name: Fetch automake - if: ${{ steps.paths.outputs.automake_SOURCE }} - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests automake - - name: Fetch libtool - if: ${{ steps.paths.outputs.libtool_SOURCE }} - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests libtool - - name: Fetch python-3_14 - if: ${{ steps.paths.outputs.python-3_14_SOURCE }} - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests python-3_14 - - name: Restore ninja from cache - id: restore_ninja - if: ${{ steps.paths.outputs.ninja_SOURCE }} - uses: actions/cache/restore@v4 - with: - path: ${{ steps.paths.outputs.ninja_INSTALL }} - key: ${{ steps.paths.outputs.ninja_CACHE_KEY }}-install - - name: Build ninja - if: ${{ steps.paths.outputs.ninja_SOURCE && ! steps.restore_ninja.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py build --no-tests ninja - - name: Save ninja to cache - uses: actions/cache/save@v4 - if: ${{ steps.paths.outputs.ninja_SOURCE && ! steps.restore_ninja.outputs.cache-hit }} - with: - path: ${{ steps.paths.outputs.ninja_INSTALL }} - key: ${{ steps.paths.outputs.ninja_CACHE_KEY }}-install - - name: Restore cmake from cache - id: restore_cmake - if: ${{ steps.paths.outputs.cmake_SOURCE }} - uses: actions/cache/restore@v4 - with: - path: ${{ steps.paths.outputs.cmake_INSTALL }} - key: ${{ steps.paths.outputs.cmake_CACHE_KEY }}-install - - name: Build cmake - if: ${{ steps.paths.outputs.cmake_SOURCE && ! steps.restore_cmake.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py build --no-tests cmake - - name: Save cmake to cache - uses: actions/cache/save@v4 - if: ${{ steps.paths.outputs.cmake_SOURCE && ! steps.restore_cmake.outputs.cache-hit }} - with: - path: ${{ steps.paths.outputs.cmake_INSTALL }} - key: ${{ steps.paths.outputs.cmake_CACHE_KEY }}-install - - name: Restore python-setuptools from cache - id: restore_python-setuptools - if: ${{ steps.paths.outputs.python-setuptools_SOURCE }} - uses: actions/cache/restore@v4 - with: - path: ${{ steps.paths.outputs.python-setuptools_INSTALL }} - key: ${{ steps.paths.outputs.python-setuptools_CACHE_KEY }}-install - - name: Build python-setuptools - if: ${{ steps.paths.outputs.python-setuptools_SOURCE && ! steps.restore_python-setuptools.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py build --no-tests python-setuptools - - name: Save python-setuptools to cache - uses: actions/cache/save@v4 - if: ${{ steps.paths.outputs.python-setuptools_SOURCE && ! steps.restore_python-setuptools.outputs.cache-hit }} - with: - path: ${{ steps.paths.outputs.python-setuptools_INSTALL }} - key: ${{ steps.paths.outputs.python-setuptools_CACHE_KEY }}-install - - name: Restore autoconf from cache - id: restore_autoconf - if: ${{ steps.paths.outputs.autoconf_SOURCE }} - uses: actions/cache/restore@v4 - with: - path: ${{ steps.paths.outputs.autoconf_INSTALL }} - key: ${{ steps.paths.outputs.autoconf_CACHE_KEY }}-install - - name: Build autoconf - if: ${{ steps.paths.outputs.autoconf_SOURCE && ! steps.restore_autoconf.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py build --no-tests autoconf - - name: Save autoconf to cache - uses: actions/cache/save@v4 - if: ${{ steps.paths.outputs.autoconf_SOURCE && ! steps.restore_autoconf.outputs.cache-hit }} - with: - path: ${{ steps.paths.outputs.autoconf_INSTALL }} - key: ${{ steps.paths.outputs.autoconf_CACHE_KEY }}-install - - name: Restore automake from cache - id: restore_automake - if: ${{ steps.paths.outputs.automake_SOURCE }} - uses: actions/cache/restore@v4 - with: - path: ${{ steps.paths.outputs.automake_INSTALL }} - key: ${{ steps.paths.outputs.automake_CACHE_KEY }}-install - - name: Build automake - if: ${{ steps.paths.outputs.automake_SOURCE && ! steps.restore_automake.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py build --no-tests automake - - name: Save automake to cache - uses: actions/cache/save@v4 - if: ${{ steps.paths.outputs.automake_SOURCE && ! steps.restore_automake.outputs.cache-hit }} - with: - path: ${{ steps.paths.outputs.automake_INSTALL }} - key: ${{ steps.paths.outputs.automake_CACHE_KEY }}-install - - name: Restore libtool from cache - id: restore_libtool - if: ${{ steps.paths.outputs.libtool_SOURCE }} - uses: actions/cache/restore@v4 - with: - path: ${{ steps.paths.outputs.libtool_INSTALL }} - key: ${{ steps.paths.outputs.libtool_CACHE_KEY }}-install - - name: Build libtool - if: ${{ steps.paths.outputs.libtool_SOURCE && ! steps.restore_libtool.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py build --no-tests libtool - - name: Save libtool to cache - uses: actions/cache/save@v4 - if: ${{ steps.paths.outputs.libtool_SOURCE && ! steps.restore_libtool.outputs.cache-hit }} - with: - path: ${{ steps.paths.outputs.libtool_INSTALL }} - key: ${{ steps.paths.outputs.libtool_CACHE_KEY }}-install - - name: Restore python-3_14 from cache - id: restore_python-3_14 - if: ${{ steps.paths.outputs.python-3_14_SOURCE }} - uses: actions/cache/restore@v4 - with: - path: ${{ steps.paths.outputs.python-3_14_INSTALL }} - key: ${{ steps.paths.outputs.python-3_14_CACHE_KEY }}-install - - name: Build python-3_14 - if: ${{ steps.paths.outputs.python-3_14_SOURCE && ! steps.restore_python-3_14.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py build --no-tests python-3_14 - - name: Save python-3_14 to cache - uses: actions/cache/save@v4 - if: ${{ steps.paths.outputs.python-3_14_SOURCE && ! steps.restore_python-3_14.outputs.cache-hit }} - with: - path: ${{ steps.paths.outputs.python-3_14_INSTALL }} - key: ${{ steps.paths.outputs.python-3_14_CACHE_KEY }}-install - - name: Build cinderx-3_14 - run: python3 build/fbcode_builder/getdeps.py build --src-dir=. cinderx-3_14 --project-install-prefix cinderx-3_14:/usr/local - - name: Copy artifacts - run: python3 build/fbcode_builder/getdeps.py fixup-dyn-deps --strip --src-dir=. cinderx-3_14 _artifacts/linux --project-install-prefix cinderx-3_14:/usr/local --final-install-prefix /usr/local - - uses: actions/upload-artifact@v6 - with: - name: cinderx-3_14 - path: _artifacts - - name: Test cinderx-3_14 - run: python3 build/fbcode_builder/getdeps.py test --src-dir=. cinderx-3_14 --project-install-prefix cinderx-3_14:/usr/local diff --git a/.github/workflows/getdeps-main-linux.yml b/.github/workflows/getdeps-main-linux.yml deleted file mode 100644 index 448602dc0..000000000 --- a/.github/workflows/getdeps-main-linux.yml +++ /dev/null @@ -1,166 +0,0 @@ -# This file was @generated by getdeps.py - -name: getdeps-main-Linux - -on: - push: - branches: - - main - pull_request: - branches: - - main - -permissions: - contents: read # to fetch code (actions/checkout) - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - id: paths - name: Query paths - run: python3 build/fbcode_builder/getdeps.py query-paths --recursive --src-dir=. cinderx-main >> "$GITHUB_OUTPUT" - - name: Fetch ninja - if: ${{ steps.paths.outputs.ninja_SOURCE }} - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests ninja - - name: Fetch cmake - if: ${{ steps.paths.outputs.cmake_SOURCE }} - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests cmake - - name: Fetch python-setuptools - if: ${{ steps.paths.outputs.python-setuptools_SOURCE }} - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests python-setuptools - - name: Fetch autoconf - if: ${{ steps.paths.outputs.autoconf_SOURCE }} - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests autoconf - - name: Fetch automake - if: ${{ steps.paths.outputs.automake_SOURCE }} - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests automake - - name: Fetch libtool - if: ${{ steps.paths.outputs.libtool_SOURCE }} - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests libtool - - name: Fetch python-main - if: ${{ steps.paths.outputs.python-main_SOURCE }} - run: python3 build/fbcode_builder/getdeps.py fetch --no-tests python-main - - name: Restore ninja from cache - id: restore_ninja - if: ${{ steps.paths.outputs.ninja_SOURCE }} - uses: actions/cache/restore@v4 - with: - path: ${{ steps.paths.outputs.ninja_INSTALL }} - key: ${{ steps.paths.outputs.ninja_CACHE_KEY }}-install - - name: Build ninja - if: ${{ steps.paths.outputs.ninja_SOURCE && ! steps.restore_ninja.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py build --no-tests ninja - - name: Save ninja to cache - uses: actions/cache/save@v4 - if: ${{ steps.paths.outputs.ninja_SOURCE && ! steps.restore_ninja.outputs.cache-hit }} - with: - path: ${{ steps.paths.outputs.ninja_INSTALL }} - key: ${{ steps.paths.outputs.ninja_CACHE_KEY }}-install - - name: Restore cmake from cache - id: restore_cmake - if: ${{ steps.paths.outputs.cmake_SOURCE }} - uses: actions/cache/restore@v4 - with: - path: ${{ steps.paths.outputs.cmake_INSTALL }} - key: ${{ steps.paths.outputs.cmake_CACHE_KEY }}-install - - name: Build cmake - if: ${{ steps.paths.outputs.cmake_SOURCE && ! steps.restore_cmake.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py build --no-tests cmake - - name: Save cmake to cache - uses: actions/cache/save@v4 - if: ${{ steps.paths.outputs.cmake_SOURCE && ! steps.restore_cmake.outputs.cache-hit }} - with: - path: ${{ steps.paths.outputs.cmake_INSTALL }} - key: ${{ steps.paths.outputs.cmake_CACHE_KEY }}-install - - name: Restore python-setuptools from cache - id: restore_python-setuptools - if: ${{ steps.paths.outputs.python-setuptools_SOURCE }} - uses: actions/cache/restore@v4 - with: - path: ${{ steps.paths.outputs.python-setuptools_INSTALL }} - key: ${{ steps.paths.outputs.python-setuptools_CACHE_KEY }}-install - - name: Build python-setuptools - if: ${{ steps.paths.outputs.python-setuptools_SOURCE && ! steps.restore_python-setuptools.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py build --no-tests python-setuptools - - name: Save python-setuptools to cache - uses: actions/cache/save@v4 - if: ${{ steps.paths.outputs.python-setuptools_SOURCE && ! steps.restore_python-setuptools.outputs.cache-hit }} - with: - path: ${{ steps.paths.outputs.python-setuptools_INSTALL }} - key: ${{ steps.paths.outputs.python-setuptools_CACHE_KEY }}-install - - name: Restore autoconf from cache - id: restore_autoconf - if: ${{ steps.paths.outputs.autoconf_SOURCE }} - uses: actions/cache/restore@v4 - with: - path: ${{ steps.paths.outputs.autoconf_INSTALL }} - key: ${{ steps.paths.outputs.autoconf_CACHE_KEY }}-install - - name: Build autoconf - if: ${{ steps.paths.outputs.autoconf_SOURCE && ! steps.restore_autoconf.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py build --no-tests autoconf - - name: Save autoconf to cache - uses: actions/cache/save@v4 - if: ${{ steps.paths.outputs.autoconf_SOURCE && ! steps.restore_autoconf.outputs.cache-hit }} - with: - path: ${{ steps.paths.outputs.autoconf_INSTALL }} - key: ${{ steps.paths.outputs.autoconf_CACHE_KEY }}-install - - name: Restore automake from cache - id: restore_automake - if: ${{ steps.paths.outputs.automake_SOURCE }} - uses: actions/cache/restore@v4 - with: - path: ${{ steps.paths.outputs.automake_INSTALL }} - key: ${{ steps.paths.outputs.automake_CACHE_KEY }}-install - - name: Build automake - if: ${{ steps.paths.outputs.automake_SOURCE && ! steps.restore_automake.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py build --no-tests automake - - name: Save automake to cache - uses: actions/cache/save@v4 - if: ${{ steps.paths.outputs.automake_SOURCE && ! steps.restore_automake.outputs.cache-hit }} - with: - path: ${{ steps.paths.outputs.automake_INSTALL }} - key: ${{ steps.paths.outputs.automake_CACHE_KEY }}-install - - name: Restore libtool from cache - id: restore_libtool - if: ${{ steps.paths.outputs.libtool_SOURCE }} - uses: actions/cache/restore@v4 - with: - path: ${{ steps.paths.outputs.libtool_INSTALL }} - key: ${{ steps.paths.outputs.libtool_CACHE_KEY }}-install - - name: Build libtool - if: ${{ steps.paths.outputs.libtool_SOURCE && ! steps.restore_libtool.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py build --no-tests libtool - - name: Save libtool to cache - uses: actions/cache/save@v4 - if: ${{ steps.paths.outputs.libtool_SOURCE && ! steps.restore_libtool.outputs.cache-hit }} - with: - path: ${{ steps.paths.outputs.libtool_INSTALL }} - key: ${{ steps.paths.outputs.libtool_CACHE_KEY }}-install - - name: Restore python-main from cache - id: restore_python-main - if: ${{ steps.paths.outputs.python-main_SOURCE }} - uses: actions/cache/restore@v4 - with: - path: ${{ steps.paths.outputs.python-main_INSTALL }} - key: ${{ steps.paths.outputs.python-main_CACHE_KEY }}-install - - name: Build python-main - if: ${{ steps.paths.outputs.python-main_SOURCE && ! steps.restore_python-main.outputs.cache-hit }} - run: python3 build/fbcode_builder/getdeps.py build --no-tests python-main - - name: Save python-main to cache - uses: actions/cache/save@v4 - if: ${{ steps.paths.outputs.python-main_SOURCE && ! steps.restore_python-main.outputs.cache-hit }} - with: - path: ${{ steps.paths.outputs.python-main_INSTALL }} - key: ${{ steps.paths.outputs.python-main_CACHE_KEY }}-install - - name: Build cinderx-main - run: python3 build/fbcode_builder/getdeps.py build --src-dir=. cinderx-main --project-install-prefix cinderx-main:/usr/local - - name: Copy artifacts - run: python3 build/fbcode_builder/getdeps.py fixup-dyn-deps --strip --src-dir=. cinderx-main _artifacts/linux --project-install-prefix cinderx-main:/usr/local --final-install-prefix /usr/local - - uses: actions/upload-artifact@v6 - with: - name: cinderx-main - path: _artifacts - - name: Test cinderx-main - run: python3 build/fbcode_builder/getdeps.py test --src-dir=. cinderx-main --project-install-prefix cinderx-main:/usr/local diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 23935adcd..8595e1433 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -16,18 +16,21 @@ on: jobs: build_wheels: + if: github.repository == 'facebookincubator/cinderx' name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest] + os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest] steps: - - uses: actions/checkout@v5 + - name: Checkout CinderX + uses: actions/checkout@v6 - - uses: actions/setup-python@v5 - with: - python-version: '3.14.3' + # cibuildwheel uses Python from a docker container. The Python version is + # controlled by selecting the docker image version in pyproject.toml. + - name: Set up Python + uses: actions/setup-python@v6 - name: Install cibuildwheel run: python -m pip install cibuildwheel @@ -35,25 +38,27 @@ jobs: - name: Build wheels run: python -m cibuildwheel --output-dir wheelhouse env: - CIBW_BUILD: cp314-manylinux_x86_64 cp314-musllinux_x86_64 - CIBW_ENVIRONMENT: CINDERX_ENABLE_PGO=1 CINDERX_ENABLE_LTO=1 CINDERX_VERSION_PATCH=${{ inputs.patch_version || '0' }} - CIBW_BUILD_VERBOSITY: 3 + CINDERX_VERSION_PATCH: ${{ inputs.patch_version || '0' }} - - uses: actions/upload-artifact@v4 + - name: Upload wheel artifacts + uses: actions/upload-artifact@v7 with: name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} path: ./wheelhouse/*.whl build_sdist: + if: github.repository == 'facebookincubator/cinderx' name: Build source distribution runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - name: Checkout CinderX + uses: actions/checkout@v6 - - uses: actions/setup-python@v5 - with: - python-version: '3.14.3' + # An sdist is just source code and doesn't contain compiled artifacts, it + # doesn't need to be built per patch version. + - name: Set up Python + uses: actions/setup-python@v6 - name: Install build tools run: python -m pip install build @@ -63,12 +68,14 @@ jobs: env: CINDERX_VERSION_PATCH: ${{ inputs.patch_version || '0' }} - - uses: actions/upload-artifact@v4 + - name: Upload sdist artifact + uses: actions/upload-artifact@v7 with: name: cibw-sdist path: dist/*.tar.gz publish: + if: github.repository == 'facebookincubator/cinderx' name: Publish to PyPI needs: [build_wheels, build_sdist] runs-on: ubuntu-latest @@ -78,7 +85,7 @@ jobs: steps: - name: Download all artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: pattern: cibw-* path: dist diff --git a/.gitignore b/.gitignore index 5c94859e3..4c3b2edaa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ scratch cinderx.egg-info dist +__pycache__/ +# Generated by BuildPy / BuildExt (editable installs). +cinderx/PythonLib/cinderx/opcode.py diff --git a/CMakeLists.txt b/CMakeLists.txt index c76aaf49d..8f0edcf1c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,14 +1,5 @@ # (c) Meta Platforms, Inc. and affiliates. -message("=====================================================================") -message("= WARNING =") -message("=====================================================================") -message("= =") -message("= This CMake build is not well supported, and is primarily used for =") -message("= experimentation. Do not expect this to work on your machine. =") -message("=====================================================================") -message("") - cmake_minimum_required(VERSION 3.12) project(_cinderx) @@ -21,8 +12,14 @@ option(ENABLE_PGO_GENERATE "Build with PGO profile generation" OFF) option(ENABLE_PGO_USE "Build with PGO profile use" OFF) set(PGO_PROFILE_FILE "" CACHE STRING "Path to PGO profile data") +option(BUILD_RUNTIME_TESTS "Build the cinderx/RuntimeTests C++ GoogleTest binary" OFF) + if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") set(MACOS 1) + set(WINDOWS 0) +elseif(${CMAKE_SYSTEM_NAME} MATCHES "Windows") + set(MACOS 0) + set(WINDOWS 1) else() set(MACOS 0) endif() @@ -34,10 +31,17 @@ endif() set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(SHARED_FLAGS "-DPy_BUILD_CORE -DPy_BUILD_CORE_MODULE -fPIC -Wall -Wextra -Wno-c99-designator -Wno-c++11-narrowing -Wno-cast-function-type-mismatch -Wno-deprecated-declarations -Wno-missing-field-initializers -Wno-null-pointer-subtraction -Wno-sign-compare -Wno-unknown-pragmas -Wno-unused-function -Wno-unused-parameter") +if (WINDOWS) + set(SHARED_FLAGS "-DPy_BUILD_CORE -DPy_BUILD_CORE_MODULE") + set(CXX_FLAGS "") +else() + set(SHARED_FLAGS "-DPy_BUILD_CORE -DPy_BUILD_CORE_MODULE -fPIC -fvisibility=hidden -Wall -Wextra -Wno-attributes -Wno-c99-designator -Wno-c++11-narrowing -Wno-cast-function-type -Wno-cast-function-type-mismatch -Wno-comment -Wno-deprecated-declarations -Wno-missing-field-initializers -Wno-null-pointer-subtraction -Wno-parentheses -Wno-sign-compare -Wno-unknown-pragmas -Wno-unused-function -Wno-unused-parameter") + set(CXX_FLAGS "-fvisibility-inlines-hidden") +endif() macro(set_flag VAR) - if (DEFINED ${VAR} AND ${${VAR}}) + # Undefined feature flags should behave like OFF. + if (${VAR}) set(SHARED_FLAGS "-D${VAR} ${SHARED_FLAGS}") endif() endmacro() @@ -46,6 +50,9 @@ set_flag(ENABLE_ADAPTIVE_STATIC_PYTHON) set_flag(ENABLE_DISASSEMBLER) set_flag(ENABLE_ELF_READER) set_flag(ENABLE_EVAL_HOOK) +if (ENABLE_FREE_THREADING) + set(SHARED_FLAGS "-DPy_GIL_DISABLED=1 ${SHARED_FLAGS}") +endif() set_flag(ENABLE_FUNC_EVENT_MODIFY_QUALNAME) set_flag(ENABLE_GENERATOR_AWAITER) set_flag(ENABLE_INTERPRETER_LOOP) @@ -56,9 +63,10 @@ set_flag(ENABLE_PEP523_HOOK) set_flag(ENABLE_PERF_TRAMPOLINE) set_flag(ENABLE_SYMBOLIZER) set_flag(ENABLE_USDT) +set_flag(ENABLE_ZLIB) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SHARED_FLAGS}") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SHARED_FLAGS}") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_FLAGS} ${SHARED_FLAGS}") ############################################################################## # Apply LTO (Link-Time Optimization) if enabled @@ -66,36 +74,63 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SHARED_FLAGS}") option(ENABLE_LTO "Enable Link-Time Optimization (full)" OFF) if(ENABLE_LTO) - # Only support Linux - if(MACOS) - message(FATAL_ERROR "LTO is only supported on Linux") - endif() - - message(STATUS "LTO: Enabled (full LTO)") + message(STATUS "LTO: Enabled") # Detect compiler type if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(USING_CLANG TRUE) elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU") set(USING_GCC TRUE) + elseif(MSVC) + set(USING_MSVC TRUE) else() - message(FATAL_ERROR "LTO is only supported with Clang or GCC compilers") + message(FATAL_ERROR "LTO is only supported with Clang, GCC, or MSVC compilers") endif() - # For Clang, we need llvm-ar instead of regular ar + # For Clang LTO we need an LLVM archiver. Use llvm-lib on Windows (clang-cl + # wants MSVC-style /flags, which llvm-ar rejects), and llvm-ar elsewhere. + # Both are optional on macOS since AppleClang may work with the system ar. if(USING_CLANG) - find_program(LLVM_AR llvm-ar) - if(NOT LLVM_AR) - message(FATAL_ERROR "llvm-ar is required for LTO with Clang but was not found") + if(WINDOWS) + find_program(LLVM_LIB llvm-lib) + if(LLVM_LIB) + set(CMAKE_AR ${LLVM_LIB}) + message(STATUS "LTO: Using llvm-lib: ${CMAKE_AR}") + else() + message(STATUS + "LTO: Using system lib (llvm-lib not found, Clang-CL may work with" + " system lib)") + endif() + else() + find_program(LLVM_AR llvm-ar) + if(LLVM_AR) + set(CMAKE_AR ${LLVM_AR}) + message(STATUS "LTO: Using llvm-ar: ${CMAKE_AR}") + elseif(MACOS) + message(STATUS + "LTO: Using system ar (llvm-ar not found," + " AppleClang may work with system ar)") + else() + message(FATAL_ERROR "llvm-ar is required for LTO with Clang but was not found") + endif() endif() - set(CMAKE_AR ${LLVM_AR}) - message(STATUS "LTO: Using llvm-ar: ${CMAKE_AR}") - set(LTO_FLAG "-flto") - set(LTO_LINKER_FLAGS "-flto") + # Use ThinLTO on macOS for better performance and lower memory usage. + if(MACOS) + set(LTO_FLAG "-flto=thin") + set(LTO_LINKER_FLAGS "-flto=thin") + else() + set(LTO_FLAG "-flto") + set(LTO_LINKER_FLAGS "-flto") + endif() elseif(USING_GCC) set(LTO_FLAG "-flto") set(LTO_LINKER_FLAGS "-flto -fuse-linker-plugin -ffat-lto-objects") + elseif(USING_MSVC) + # MSVC uses /GL for compilation and /LTCG for linking + # Alternatively, we can use CMake's INTERPROCEDURAL_OPTIMIZATION + set(LTO_FLAG "/GL") + set(LTO_LINKER_FLAGS "/LTCG") endif() # Apply flags to all C and C++ compilation @@ -158,21 +193,24 @@ include(FetchContent) find_package(Python ${PY_VERSION} EXACT COMPONENTS Interpreter Development.Module REQUIRED) +# RuntimeTests embeds Python, so it needs Python::Python. +if (BUILD_RUNTIME_TESTS) + find_package(Python ${PY_VERSION} EXACT COMPONENTS Development.Embed REQUIRED) +endif() + # Some of our files are partially generated from CPython source, and they # expect to be able to include files relative to Include/internal. set(Python_INCLUDE_DIRS ${Python_INCLUDE_DIRS} "${Python_INCLUDE_DIRS}/internal") ######################################## -# asmjit +# asmjit (vendored) set(ASMJIT_STATIC TRUE) - -FetchContent_Declare( - asmjit - GIT_REPOSITORY https://github.com/asmjit/asmjit - GIT_TAG cecc73f2979e9704c81a2c2ec79a7475b31c56ac # 2025-May-10 -) -FetchContent_MakeAvailable(asmjit) +set(ASMJIT_NO_CUSTOM_FLAGS TRUE) +add_subdirectory(${PROJECT_SOURCE_DIR}/ThirdParty/asmjit asmjit) +if (NOT WINDOWS) + target_compile_options(asmjit PRIVATE -Wno-class-memaccess) +endif() ######################################## # fmt @@ -180,7 +218,7 @@ FetchContent_MakeAvailable(asmjit) FetchContent_Declare( fmt GIT_REPOSITORY https://github.com/fmtlib/fmt - GIT_TAG 11.2.0 # 2025-May-03 + GIT_TAG 12.1.0 # 2025-Oct-29 ) FetchContent_MakeAvailable(fmt) @@ -212,7 +250,60 @@ file(COPY ${usdt_SOURCE_DIR}/usdt.h DESTINATION ${USDT_DIR}) ######################################## # zlib -find_package(ZLIB) +if (${ENABLE_ZLIB}) + FetchContent_Declare( + zlib + GIT_REPOSITORY "https://github.com/madler/zlib.git" + FIND_PACKAGE_ARGS NAMES ZLIB + ) + find_package(ZLIB REQUIRED) +endif() + +######################################## +# capstone (optional, for JIT disassembly) + +if (${ENABLE_DISASSEMBLER}) + FetchContent_Declare( + capstone + GIT_REPOSITORY https://github.com/capstone-engine/capstone + GIT_TAG 5.0.7 + ) + + # Skip unsupported architectures. + set(CAPSTONE_ARM_SUPPORT OFF CACHE BOOL "" FORCE) + set(CAPSTONE_BPF_SUPPORT OFF CACHE BOOL "" FORCE) + set(CAPSTONE_BUILD_CSTOOL OFF CACHE BOOL "" FORCE) + set(CAPSTONE_BUILD_TESTS OFF CACHE BOOL "" FORCE) + set(CAPSTONE_EVM_SUPPORT OFF CACHE BOOL "" FORCE) + set(CAPSTONE_M680X_SUPPORT OFF CACHE BOOL "" FORCE) + set(CAPSTONE_M68K_SUPPORT OFF CACHE BOOL "" FORCE) + set(CAPSTONE_MIPS_SUPPORT OFF CACHE BOOL "" FORCE) + set(CAPSTONE_MOS65XX_SUPPORT OFF CACHE BOOL "" FORCE) + set(CAPSTONE_PPC_SUPPORT OFF CACHE BOOL "" FORCE) + set(CAPSTONE_RISCV_SUPPORT OFF CACHE BOOL "" FORCE) + set(CAPSTONE_SH_SUPPORT OFF CACHE BOOL "" FORCE) + set(CAPSTONE_SPARC_SUPPORT OFF CACHE BOOL "" FORCE) + set(CAPSTONE_SYSZ_SUPPORT OFF CACHE BOOL "" FORCE) + set(CAPSTONE_TMS320C64X_SUPPORT OFF CACHE BOOL "" FORCE) + set(CAPSTONE_TRICORE_SUPPORT OFF CACHE BOOL "" FORCE) + set(CAPSTONE_WASM_SUPPORT OFF CACHE BOOL "" FORCE) + set(CAPSTONE_XCORE_SUPPORT OFF CACHE BOOL "" FORCE) + + # x86-64 and aarch64 are supported. + if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + set(CAPSTONE_X86_SUPPORT ON CACHE BOOL "" FORCE) + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64") + set(CAPSTONE_AARCH64_SUPPORT ON CACHE BOOL "" FORCE) + endif() + + FetchContent_MakeAvailable(capstone) + target_compile_options(capstone PRIVATE -w) + target_compile_options(capstone_static PRIVATE -w) + # CAPSTONE_STATIC suppresses CAPSTONE_EXPORT's visibility("default") annotation, + # so capstone symbols remain hidden when we build with -fvisibility=hidden. + target_compile_definitions(capstone PRIVATE CAPSTONE_STATIC) + target_compile_definitions(capstone_static PRIVATE CAPSTONE_STATIC) +endif() ############################################################################## # Build cinderx @@ -228,15 +319,23 @@ include_directories( ######################################## # Common/ -file(GLOB_RECURSE COMMON_SOURCES ${PROJECT_SOURCE_DIR}/Common/*.cpp) +file(GLOB_RECURSE COMMON_SOURCES ${PROJECT_SOURCE_DIR}/Common/*.cpp ${PROJECT_SOURCE_DIR}/Common/*.c) add_library(common ${COMMON_SOURCES}) -target_link_libraries(common PRIVATE asmjit::asmjit fmt::fmt ZLIB::ZLIB) +if (${ENABLE_ZLIB}) + target_link_libraries(common PRIVATE asmjit::asmjit fmt::fmt ZLIB::ZLIB) +else() + target_link_libraries(common PRIVATE asmjit::asmjit fmt::fmt) +endif() ######################################## # UpstreamBorrow/ if (${PY_VERSION} EQUAL 3.12 OR ${PY_VERSION} EQUAL 3.14 OR ${PY_VERSION} EQUAL 3.15) - set(BORROWED_C ${PROJECT_SOURCE_DIR}/UpstreamBorrow/borrowed-${PY_VERSION}.gen_cached.c) + if (ENABLE_FREE_THREADING) + set(BORROWED_C ${PROJECT_SOURCE_DIR}/UpstreamBorrow/borrowed-${PY_VERSION}.free-threading.gen_cached.c) + else() + set(BORROWED_C ${PROJECT_SOURCE_DIR}/UpstreamBorrow/borrowed-${PY_VERSION}.gen_cached.c) + endif() else() set(BORROWED_C "${GENERATED_HEADER_DIR}/dummy-borrowed.c") file(WRITE ${BORROWED_C} "") @@ -315,13 +414,13 @@ target_link_libraries(static-python PRIVATE fmt::fmt asmjit::asmjit common borro # Jit/ file(GLOB_RECURSE JIT_SOURCES ${PROJECT_SOURCE_DIR}/Jit/*.cpp ${PROJECT_SOURCE_DIR}/Jit/*.c) -list(FILTER JIT_SOURCES EXCLUDE REGEX ".*\.gen_cached\.c") -if (${PY_VERSION} EQUAL 3.12 OR ${PY_VERSION} EQUAL 3.14 OR ${PY_VERSION} EQUAL 3.15) - list(APPEND JIT_SOURCES "${PROJECT_SOURCE_DIR}/Jit/generators_borrowed_${PY_VERSION}.gen_cached.c") -endif() add_library(jit ${JIT_SOURCES}) +target_compile_options(jit PRIVATE -Wno-free-nonheap-object) target_link_libraries(jit PRIVATE asmjit::asmjit interpreter fmt::fmt) +if (${ENABLE_DISASSEMBLER}) + target_link_libraries(jit PRIVATE capstone_static) +endif() ######################################## # ParallelGC/ @@ -337,9 +436,8 @@ add_library(parallel-gc ${PARALLEL_GC_SOURCES}) ######################################## # _cinderx.cpp -set(SOURCES +set(CINDERX_MODULE_LIB_SOURCES # Manually listed out to not accidentally include stuff in .git. - ${PROJECT_SOURCE_DIR}/_cinderx.cpp ${PROJECT_SOURCE_DIR}/_cinderx-lib.cpp ${PROJECT_SOURCE_DIR}/async_lazy_value.cpp ${PROJECT_SOURCE_DIR}/module_state.cpp @@ -347,15 +445,21 @@ set(SOURCES ${PROJECT_SOURCE_DIR}/python_runtime.cpp ) -add_library(${PROJECT_NAME} SHARED ${SOURCES}) - +add_library(cinderx-lib STATIC ${CINDERX_MODULE_LIB_SOURCES}) target_link_libraries( - ${PROJECT_NAME} + cinderx-lib PRIVATE Python::Module borrowed cached-properties common immortalize interpreter jit parallel-gc static-python asmjit::asmjit fmt::fmt) +add_library(${PROJECT_NAME} SHARED ${PROJECT_SOURCE_DIR}/_cinderx.cpp) + +target_link_libraries( + ${PROJECT_NAME} + PRIVATE + cinderx-lib) + # macOS doesn't allow depending on other shared libraries by default. if (${MACOS}) target_link_options(${PROJECT_NAME} PRIVATE -undefined dynamic_lookup) @@ -363,4 +467,83 @@ endif() set_target_properties(${PROJECT_NAME} PROPERTIES PREFIX "") set_target_properties(${PROJECT_NAME} PROPERTIES SUFFIX "") -set_target_properties(${PROJECT_NAME} PROPERTIES OUTPUT_NAME "_cinderx.so") +if (WINDOWS) + set_target_properties(${PROJECT_NAME} PROPERTIES OUTPUT_NAME "_cinderx.pyd") +else() + set_target_properties(${PROJECT_NAME} PROPERTIES OUTPUT_NAME "_cinderx.so") +endif() + +############################################################################## +# RuntimeTests/ — opt-in C++ GoogleTest binary. + +if (BUILD_RUNTIME_TESTS) + enable_testing() + + ######################################## + # GoogleTest + + FetchContent_Declare( + googletest + GIT_REPOSITORY https://github.com/google/googletest + GIT_TAG v1.17.0 # 2025-Apr-30 + ) + FetchContent_MakeAvailable(googletest) + + ######################################## + # RuntimeTests executable + + file(GLOB RUNTIME_TESTS_SOURCES + CONFIGURE_DEPENDS + ${PROJECT_SOURCE_DIR}/RuntimeTests/*.cpp + ) + + if (NOT ENABLE_ELF_READER) + list(REMOVE_ITEM RUNTIME_TESTS_SOURCES ${PROJECT_SOURCE_DIR}/RuntimeTests/elf_test.cpp) + endif() + + if (NOT ENABLE_SYMBOLIZER) + list(REMOVE_ITEM RUNTIME_TESTS_SOURCES ${PROJECT_SOURCE_DIR}/RuntimeTests/util_test.cpp) + endif() + + # These tests walk native frame-pointer chains in their own fixture code. + if (CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC") + # clang-cl's driver takes MSVC's spelling of this; the clang one has to be + # smuggled past it with /clang: and is not worth the indirection. + set(STACK_WALK_TEST_FLAGS /Oy-) + elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + set(STACK_WALK_TEST_FLAGS -fno-omit-frame-pointer) + endif() + if (STACK_WALK_TEST_FLAGS) + set_source_files_properties( + ${PROJECT_SOURCE_DIR}/RuntimeTests/stack_walk_test.cpp + PROPERTIES COMPILE_OPTIONS "${STACK_WALK_TEST_FLAGS}" + ) + endif() + + add_executable(RuntimeTests ${RUNTIME_TESTS_SOURCES}) + + target_link_libraries( + RuntimeTests + PRIVATE + cinderx-lib + asmjit::asmjit fmt::fmt + GTest::gtest + GTest::gmock + Python::Python + ) + + target_compile_definitions( + RuntimeTests + PRIVATE + CINDERX_RUNTIME_TESTS_STATIC_CINDERX=1 + # Used by RuntimeTest::SetUp() to import cinderx from PythonLib. + CINDERX_RUNTIME_TESTS_PYTHONPATH_PACKAGE=${PROJECT_SOURCE_DIR}/PythonLib + ) + + add_test( + NAME RuntimeTests + COMMAND RuntimeTests + # RuntimeTests open source-tree test data relative to cwd. + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + ) +endif() diff --git a/CMakeSettings.json b/CMakeSettings.json new file mode 100644 index 000000000..4bcdbe9d0 --- /dev/null +++ b/CMakeSettings.json @@ -0,0 +1,25 @@ +{ + "configurations": [ + { + "name": "x64-Clang-Debug", + "generator": "Ninja", + "configurationType": "Debug", + "buildRoot": "${projectDir}\\out\\build\\${name}", + "installRoot": "${projectDir}\\out\\install\\${name}", + "cmakeCommandArgs": "-DPY_VERSION=3.14 -DENABLE_ADAPTIVE_STATIC_PYTHON=0 -DENABLE_DISASSEMBLER=0 -DENABLE_ELF_READER=0 -DENABLE_EVAL_HOOK=0 -DENABLE_FUNC_EVENT_MODIFY_QUALNAME=0 -DENABLE_GENERATOR_AWAITER=0 -DENABLE_INTERPRETER_LOOP=1 -DENABLE_LAZY_IMPORTS=0 -DENABLE_LIGHTWEIGHT_FRAMES=0 -DENABLE_PARALLEL_GC=0 -DENABLE_PEP523_HOOK=1 -DENABLE_PERF_TRAMPOLINE=0 -DENABLE_SYMBOLIZER=0 -DENABLE_USDT=0 -DENABLE_ZLIB=0", + "buildCommandArgs": "", + "ctestCommandArgs": "", + "inheritEnvironments": [ "clang_cl_x64_x64" ] + }, + { + "name": "x64-Clang-Release", + "generator": "Ninja", + "configurationType": "RelWithDebInfo", + "buildRoot": "${projectDir}\\out\\build\\${name}", + "installRoot": "${projectDir}\\out\\install\\${name}", + "cmakeCommandArgs": "-DPY_VERSION=3.14 -DENABLE_ADAPTIVE_STATIC_PYTHON=0 -DENABLE_DISASSEMBLER=0 -DENABLE_ELF_READER=0 -DENABLE_EVAL_HOOK=0 -DENABLE_FUNC_EVENT_MODIFY_QUALNAME=0 -DENABLE_GENERATOR_AWAITER=0 -DENABLE_INTERPRETER_LOOP=1 -DENABLE_LAZY_IMPORTS=0 -DENABLE_LIGHTWEIGHT_FRAMES=0 -DENABLE_PARALLEL_GC=0 -DENABLE_PEP523_HOOK=1 -DENABLE_PERF_TRAMPOLINE=0 -DENABLE_SYMBOLIZER=0 -DENABLE_USDT=0 -DENABLE_ZLIB=0", + "ctestCommandArgs": "", + "inheritEnvironments": [ "clang_cl_x64_x64" ] + } + ] +} \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in index 81bd5a3d2..fea605290 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ include CMakeLists.txt -recursive-include cinderx *.cpp *.c *.h +recursive-include cinderx *.cpp *.c *.h CMakeLists.txt # Opcode directories have invalid Python package names (e.g. opcodes/3.14) so # they have to be included manually. diff --git a/README.md b/README.md index d52393f18..ce16f31bb 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ [![PyPI - Version](https://img.shields.io/pypi/v/cinderx.svg)](https://pypi.org/pypi/cinderx/) +![The CinderX logo, which is a lowercase "cinderx" with the dot in the i as a small flame, and a stylized orange x](assets/png/logo.png) + CinderX is a Python extension that improves the performance of the Python runtime. @@ -24,12 +26,13 @@ However these features are not compatible with the stock CPython runtime yet. ## Requirements -- Python 3.14 or later -- Linux (x86_64) +- Python 3.14 - GCC 13+ or Clang 18+ -The extension should build and import on macOS but most features will be -disabled at runtime. Windows is not yet supported at all. +| | Linux | macOS | Windows | +| ------- | ------------------ | ------------------ | ------------------ | +| x86-64 | :white_check_mark: | :x: | :white_check_mark: | +| aarch64 | :white_check_mark: | :white_check_mark: | :x: | ## Installation @@ -37,6 +40,24 @@ disabled at runtime. Windows is not yet supported at all. pip install cinderx ``` +## Using the JIT + +The recommended way to start using the JIT is to do: + +```python +import cinderx.jit + +cinderx.jit.auto() +``` + +This will configure the CinderX extension to automatically compile Python +functions to machine code. It will track what functions are called frequently +and compile the hottest ones automatically. + +See [the JIT documentation](https://facebookincubator.github.io/cinderx/jit) for +more details, or browse the full [CinderX documentation +site](https://facebookincubator.github.io/cinderx/). + ## CinderX vs Cinder [Cinder](https://github.com/facebookincubator/cinder) was a fork of the CPython @@ -46,7 +67,7 @@ decided to turn it into a Python extension to improve compatibility with newer Python versions. This extension is now known as CinderX ("the X" is for "extension"). -For Python versions 3.10 through 3.12, CinderX still depends on patches to +Historically, for Python versions 3.10 through 3.12, CinderX depended on patches to Meta's fork of the Python runtime. Python 3.14 is the first version of stock CPython that CinderX supports. diff --git a/assets/png/logo.png b/assets/png/logo.png new file mode 100644 index 000000000..7a4363c2a Binary files /dev/null and b/assets/png/logo.png differ diff --git a/build/fbcode_builder/.gitignore b/build/fbcode_builder/.gitignore deleted file mode 100644 index b98f3edfa..000000000 --- a/build/fbcode_builder/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Facebook-internal CI builds don't have write permission outside of the -# source tree, so we install all projects into this directory. -/facebook_ci -__pycache__/ -*.pyc diff --git a/build/fbcode_builder/CMake/FBBuildOptions.cmake b/build/fbcode_builder/CMake/FBBuildOptions.cmake deleted file mode 100644 index e2fcf69ca..000000000 --- a/build/fbcode_builder/CMake/FBBuildOptions.cmake +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. - -function (fb_activate_static_library_option) - option(USE_STATIC_DEPS_ON_UNIX - "If enabled, use static dependencies on unix systems. This is generally discouraged." - OFF - ) - # Mark USE_STATIC_DEPS_ON_UNIX as an "advanced" option, since enabling it - # is generally discouraged. - mark_as_advanced(USE_STATIC_DEPS_ON_UNIX) - - if(UNIX AND USE_STATIC_DEPS_ON_UNIX) - SET(CMAKE_FIND_LIBRARY_SUFFIXES ".a" PARENT_SCOPE) - endif() - - option(PREFER_STATIC_DEPS_ON_UNIX - "If enabled, use static dependencies on unix systems as possible as we can. This is generally discouraged." - OFF - ) - # Mark PREFER_STATIC_DEPS_ON_UNIX as an "advanced" option, since enabling it - # is generally discouraged. - mark_as_advanced(PREFER_STATIC_DEPS_ON_UNIX) - - if(UNIX AND PREFER_STATIC_DEPS_ON_UNIX) - SET(CMAKE_FIND_LIBRARY_SUFFIXES ".a" ".so" PARENT_SCOPE) - endif() -endfunction() diff --git a/build/fbcode_builder/CMake/FBCMakeParseArgs.cmake b/build/fbcode_builder/CMake/FBCMakeParseArgs.cmake deleted file mode 100644 index 933180189..000000000 --- a/build/fbcode_builder/CMake/FBCMakeParseArgs.cmake +++ /dev/null @@ -1,141 +0,0 @@ -# -# Copyright (c) Facebook, Inc. and its affiliates. -# -# Helper function for parsing arguments to a CMake function. -# -# This function is very similar to CMake's built-in cmake_parse_arguments() -# function, with some improvements: -# - This function correctly handles empty arguments. (cmake_parse_arguments() -# ignores empty arguments.) -# - If a multi-value argument is specified more than once, the subsequent -# arguments are appended to the original list rather than replacing it. e.g. -# if "SOURCES" is a multi-value argument, and the argument list contains -# "SOURCES a b c SOURCES x y z" then the resulting value for SOURCES will be -# "a;b;c;x;y;z" rather than "x;y;z" -# - This function errors out by default on unrecognized arguments. You can -# pass in an extra "ALLOW_UNPARSED_ARGS" argument to make it behave like -# cmake_parse_arguments(), and return the unparsed arguments in a -# _UNPARSED_ARGUMENTS variable instead. -# -# It does look like cmake_parse_arguments() handled empty arguments correctly -# from CMake 3.0 through 3.3, but it seems like this was probably broken when -# it was turned into a built-in function in CMake 3.4. Here is discussion and -# patches that fixed this behavior prior to CMake 3.0: -# https://cmake.org/pipermail/cmake-developers/2013-November/020607.html -# -# The one downside to this function over the built-in cmake_parse_arguments() -# is that I don't think we can achieve the PARSE_ARGV behavior in a non-builtin -# function, so we can't properly handle arguments that contain ";". CMake will -# treat the ";" characters as list element separators, and treat it as multiple -# separate arguments. -# -function(fb_cmake_parse_args PREFIX OPTIONS ONE_VALUE_ARGS MULTI_VALUE_ARGS ARGS) - foreach(option IN LISTS ARGN) - if ("${option}" STREQUAL "ALLOW_UNPARSED_ARGS") - set(ALLOW_UNPARSED_ARGS TRUE) - else() - message( - FATAL_ERROR - "unknown optional argument for fb_cmake_parse_args(): ${option}" - ) - endif() - endforeach() - - # Define all options as FALSE in the parent scope to start with - foreach(var_name IN LISTS OPTIONS) - set("${PREFIX}_${var_name}" "FALSE" PARENT_SCOPE) - endforeach() - - # TODO: We aren't extremely strict about error checking for one-value - # arguments here. e.g., we don't complain if a one-value argument is - # followed by another option/one-value/multi-value name rather than an - # argument. We also don't complain if a one-value argument is the last - # argument and isn't followed by a value. - - list(APPEND all_args ${ONE_VALUE_ARGS}) - list(APPEND all_args ${MULTI_VALUE_ARGS}) - set(current_variable) - set(unparsed_args) - foreach(arg IN LISTS ARGS) - list(FIND OPTIONS "${arg}" opt_index) - if("${opt_index}" EQUAL -1) - list(FIND all_args "${arg}" arg_index) - if("${arg_index}" EQUAL -1) - # This argument does not match an argument name, - # must be an argument value - if("${current_variable}" STREQUAL "") - list(APPEND unparsed_args "${arg}") - else() - # Ugh, CMake lists have a pretty fundamental flaw: they cannot - # distinguish between an empty list and a list with a single empty - # element. We track our own SEEN_VALUES_arg setting to help - # distinguish this and behave properly here. - if ("${SEEN_${current_variable}}" AND "${${current_variable}}" STREQUAL "") - set("${current_variable}" ";${arg}") - else() - list(APPEND "${current_variable}" "${arg}") - endif() - set("SEEN_${current_variable}" TRUE) - endif() - else() - # We found a single- or multi-value argument name - set(current_variable "VALUES_${arg}") - set("SEEN_${arg}" TRUE) - endif() - else() - # We found an option variable - set("${PREFIX}_${arg}" "TRUE" PARENT_SCOPE) - set(current_variable) - endif() - endforeach() - - foreach(arg_name IN LISTS ONE_VALUE_ARGS) - if(NOT "${SEEN_${arg_name}}") - unset("${PREFIX}_${arg_name}" PARENT_SCOPE) - elseif(NOT "${SEEN_VALUES_${arg_name}}") - # If the argument was seen but a value wasn't specified, error out. - # We require exactly one value to be specified. - message( - FATAL_ERROR "argument ${arg_name} was specified without a value" - ) - else() - list(LENGTH "VALUES_${arg_name}" num_args) - if("${num_args}" EQUAL 0) - # We know an argument was specified and that we called list(APPEND). - # If CMake thinks the list is empty that means there is really a single - # empty element in the list. - set("${PREFIX}_${arg_name}" "" PARENT_SCOPE) - elseif("${num_args}" EQUAL 1) - list(GET "VALUES_${arg_name}" 0 arg_value) - set("${PREFIX}_${arg_name}" "${arg_value}" PARENT_SCOPE) - else() - message( - FATAL_ERROR "too many arguments specified for ${arg_name}: " - "${VALUES_${arg_name}}" - ) - endif() - endif() - endforeach() - - foreach(arg_name IN LISTS MULTI_VALUE_ARGS) - # If this argument name was never seen, then unset the parent scope - if (NOT "${SEEN_${arg_name}}") - unset("${PREFIX}_${arg_name}" PARENT_SCOPE) - else() - # TODO: Our caller still won't be able to distinguish between an empty - # list and a list with a single empty element. We can tell which is - # which, but CMake lists don't make it easy to show this to our caller. - set("${PREFIX}_${arg_name}" "${VALUES_${arg_name}}" PARENT_SCOPE) - endif() - endforeach() - - # By default we fatal out on unparsed arguments, but return them to the - # caller if ALLOW_UNPARSED_ARGS was specified. - if (DEFINED unparsed_args) - if ("${ALLOW_UNPARSED_ARGS}") - set("${PREFIX}_UNPARSED_ARGUMENTS" "${unparsed_args}" PARENT_SCOPE) - else() - message(FATAL_ERROR "unrecognized arguments: ${unparsed_args}") - endif() - endif() -endfunction() diff --git a/build/fbcode_builder/CMake/FBCompilerSettings.cmake b/build/fbcode_builder/CMake/FBCompilerSettings.cmake deleted file mode 100644 index 585c95320..000000000 --- a/build/fbcode_builder/CMake/FBCompilerSettings.cmake +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. - -# This file applies common compiler settings that are shared across -# a number of Facebook opensource projects. -# Please use caution and your best judgement before making changes -# to these shared compiler settings in order to avoid accidentally -# breaking a build in another project! - -if (WIN32) - include(FBCompilerSettingsMSVC) -else() - include(FBCompilerSettingsUnix) -endif() diff --git a/build/fbcode_builder/CMake/FBCompilerSettingsMSVC.cmake b/build/fbcode_builder/CMake/FBCompilerSettingsMSVC.cmake deleted file mode 100644 index 932193a62..000000000 --- a/build/fbcode_builder/CMake/FBCompilerSettingsMSVC.cmake +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. - -# This file applies common compiler settings that are shared across -# a number of Facebook opensource projects. -# Please use caution and your best judgement before making changes -# to these shared compiler settings in order to avoid accidentally -# breaking a build in another project! - -add_compile_options( - /wd4250 # 'class1' : inherits 'class2::member' via dominance - /Zc:preprocessor # Enable conforming preprocessor for __VA_OPT__ support -) diff --git a/build/fbcode_builder/CMake/FBCompilerSettingsUnix.cmake b/build/fbcode_builder/CMake/FBCompilerSettingsUnix.cmake deleted file mode 100644 index c26ce78b1..000000000 --- a/build/fbcode_builder/CMake/FBCompilerSettingsUnix.cmake +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. - -# This file applies common compiler settings that are shared across -# a number of Facebook opensource projects. -# Please use caution and your best judgement before making changes -# to these shared compiler settings in order to avoid accidentally -# breaking a build in another project! - -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -Wextra -Wno-deprecated -Wno-deprecated-declarations") diff --git a/build/fbcode_builder/CMake/FBPythonBinary.cmake b/build/fbcode_builder/CMake/FBPythonBinary.cmake deleted file mode 100644 index 69e78d61e..000000000 --- a/build/fbcode_builder/CMake/FBPythonBinary.cmake +++ /dev/null @@ -1,704 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. - -include(FBCMakeParseArgs) - -# -# This file contains helper functions for building self-executing Python -# binaries. -# -# This is somewhat different than typical python installation with -# distutils/pip/virtualenv/etc. We primarily want to build a standalone -# executable, isolated from other Python packages on the system. We don't want -# to install files into the standard library python paths. This is more -# similar to PEX (https://github.com/pantsbuild/pex) and XAR -# (https://github.com/facebookincubator/xar). (In the future it would be nice -# to update this code to also support directly generating XAR files if XAR is -# available.) -# -# We also want to be able to easily define "libraries" of python files that can -# be shared and re-used between these standalone python executables, and can be -# shared across projects in different repositories. This means that we do need -# a way to "install" libraries so that they are visible to CMake builds in -# other repositories, without actually installing them in the standard python -# library paths. -# - -# If the caller has not already found Python, do so now. -# If we fail to find python now we won't fail immediately, but -# add_fb_python_executable() or add_fb_python_library() will fatal out if they -# are used. -if(NOT TARGET Python3::Interpreter) - # CMake 3.12+ ships with a FindPython3.cmake module. Try using it first. - # We find with QUIET here, since otherwise this generates some noisy warnings - # on versions of CMake before 3.12 - if (WIN32) - # On Windows we need both the Interpreter as well as the Development - # libraries. - find_package(Python3 COMPONENTS Interpreter Development QUIET) - else() - find_package(Python3 COMPONENTS Interpreter QUIET) - endif() - if(Python3_Interpreter_FOUND) - message(STATUS "Found Python 3: ${Python3_EXECUTABLE}") - else() - # Try with the FindPythonInterp.cmake module available in older CMake - # versions. Check to see if the caller has already searched for this - # themselves first. - if(NOT PYTHONINTERP_FOUND) - set(Python_ADDITIONAL_VERSIONS 3 3.6 3.5 3.4 3.3 3.2 3.1) - find_package(PythonInterp) - # TODO: On Windows we require the Python libraries as well. - # We currently do not search for them on this code path. - # For now we require building with CMake 3.12+ on Windows, so that the - # FindPython3 code path above is available. - endif() - if(PYTHONINTERP_FOUND) - if("${PYTHON_VERSION_MAJOR}" GREATER_EQUAL 3) - set(Python3_EXECUTABLE "${PYTHON_EXECUTABLE}") - add_custom_target(Python3::Interpreter) - else() - string( - CONCAT FBPY_FIND_PYTHON_ERR - "found Python ${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}, " - "but need Python 3" - ) - endif() - endif() - endif() -endif() - -# Find our helper program. -# We typically install this in the same directory as this .cmake file. -find_program( - FB_MAKE_PYTHON_ARCHIVE "make_fbpy_archive.py" - PATHS ${CMAKE_MODULE_PATH} -) -set(FB_PY_TEST_MAIN "${CMAKE_CURRENT_LIST_DIR}/fb_py_test_main.py") -set( - FB_PY_TEST_DISCOVER_SCRIPT - "${CMAKE_CURRENT_LIST_DIR}/FBPythonTestAddTests.cmake" -) -set( - FB_PY_WIN_MAIN_C - "${CMAKE_CURRENT_LIST_DIR}/fb_py_win_main.c" -) - -# An option to control the default installation location for -# install_fb_python_library(). This is relative to ${CMAKE_INSTALL_PREFIX} -set( - FBPY_LIB_INSTALL_DIR "lib/fb-py-libs" CACHE STRING - "The subdirectory where FB python libraries should be installed" -) - -# -# Build a self-executing python binary. -# -# This accepts the same arguments as add_fb_python_library(). -# -# In addition, a MAIN_MODULE argument is accepted. This argument specifies -# which module should be started as the __main__ module when the executable is -# run. If left unspecified, a __main__.py script must be present in the -# manifest. -# -function(add_fb_python_executable TARGET) - fb_py_check_available() - - # Parse the arguments - set(one_value_args BASE_DIR NAMESPACE MAIN_MODULE TYPE) - set(multi_value_args SOURCES DEPENDS) - fb_cmake_parse_args( - ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}" - ) - fb_py_process_default_args(ARG_NAMESPACE ARG_BASE_DIR) - - # Use add_fb_python_library() to perform most of our source handling - add_fb_python_library( - "${TARGET}.main_lib" - BASE_DIR "${ARG_BASE_DIR}" - NAMESPACE "${ARG_NAMESPACE}" - SOURCES ${ARG_SOURCES} - DEPENDS ${ARG_DEPENDS} - ) - - set( - manifest_files - "$" - ) - set( - source_files - "$" - ) - - # The command to build the executable archive. - # - # If we are using CMake 3.8+ we can use COMMAND_EXPAND_LISTS. - # CMP0067 isn't really the policy we care about, but seems like the best way - # to check if we are running 3.8+. - if (POLICY CMP0067) - set(extra_cmd_params COMMAND_EXPAND_LISTS) - set(make_py_args "${manifest_files}") - else() - set(extra_cmd_params) - set(make_py_args --manifest-separator "::" "$") - endif() - - set(output_file "${TARGET}${CMAKE_EXECUTABLE_SUFFIX}") - if(WIN32) - set(zipapp_output "${TARGET}.py_zipapp") - else() - set(zipapp_output "${output_file}") - endif() - set(zipapp_output_file "${zipapp_output}") - - set(is_dir_output FALSE) - if(DEFINED ARG_TYPE) - list(APPEND make_py_args "--type" "${ARG_TYPE}") - if ("${ARG_TYPE}" STREQUAL "dir") - set(is_dir_output TRUE) - # CMake doesn't really seem to like having a directory specified as an - # output; specify the __main__.py file as the output instead. - set(zipapp_output_file "${zipapp_output}/__main__.py") - # Update output_file to match zipapp_output_file for dir type - set(output_file "${zipapp_output_file}") - list(APPEND - extra_cmd_params - COMMAND "${CMAKE_COMMAND}" -E remove_directory "${zipapp_output}" - ) - endif() - endif() - - if(DEFINED ARG_MAIN_MODULE) - list(APPEND make_py_args "--main" "${ARG_MAIN_MODULE}") - endif() - - add_custom_command( - OUTPUT "${zipapp_output_file}" - ${extra_cmd_params} - COMMAND - "${Python3_EXECUTABLE}" "${FB_MAKE_PYTHON_ARCHIVE}" - -o "${zipapp_output}" - ${make_py_args} - DEPENDS - ${source_files} - "${TARGET}.main_lib.py_sources_built" - "${FB_MAKE_PYTHON_ARCHIVE}" - ) - - if(WIN32) - if(is_dir_output) - # TODO: generate a main executable that will invoke Python3 - # with the correct main module inside the output directory - else() - add_executable("${TARGET}.winmain" "${FB_PY_WIN_MAIN_C}") - target_link_libraries("${TARGET}.winmain" Python3::Python) - # The Python3::Python target doesn't seem to be set up completely - # correctly on Windows for some reason, and we have to explicitly add - # ${Python3_LIBRARY_DIRS} to the target link directories. - target_link_directories( - "${TARGET}.winmain" - PUBLIC ${Python3_LIBRARY_DIRS} - ) - add_custom_command( - OUTPUT "${output_file}" - DEPENDS "${TARGET}.winmain" "${zipapp_output_file}" - COMMAND - "cmd.exe" "/c" "copy" "/b" - "${TARGET}.winmain${CMAKE_EXECUTABLE_SUFFIX}+${zipapp_output}" - "${output_file}" - ) - endif() - endif() - - # Add an "ALL" target that depends on force ${TARGET}, - # so that ${TARGET} will be included in the default list of build targets. - add_custom_target("${TARGET}.GEN_PY_EXE" ALL DEPENDS "${output_file}") - - # Allow resolving the executable path for the target that we generate - # via a generator expression like: - # "WATCHMAN_WAIT_PATH=$" - set_property(TARGET "${TARGET}.GEN_PY_EXE" - PROPERTY EXECUTABLE "${CMAKE_CURRENT_BINARY_DIR}/${output_file}") -endfunction() - -# Define a python unittest executable. -# The executable is built using add_fb_python_executable and has the -# following differences: -# -# Each of the source files specified in SOURCES will be imported -# and have unittest discovery performed upon them. -# Those sources will be imported in the top level namespace. -# -# The ENV argument allows specifying a list of "KEY=VALUE" -# pairs that will be used by the test runner to set up the environment -# in the child process prior to running the test. This is useful for -# passing additional configuration to the test. -function(add_fb_python_unittest TARGET) - # Parse the arguments - set(multi_value_args SOURCES DEPENDS ENV PROPERTIES) - set( - one_value_args - WORKING_DIRECTORY BASE_DIR NAMESPACE TEST_LIST DISCOVERY_TIMEOUT TYPE - ) - fb_cmake_parse_args( - ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}" - ) - fb_py_process_default_args(ARG_NAMESPACE ARG_BASE_DIR) - if(NOT ARG_WORKING_DIRECTORY) - # Default the working directory to the current binary directory. - # This matches the default behavior of add_test() and other standard - # test functions like gtest_discover_tests() - set(ARG_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") - endif() - if(NOT ARG_TEST_LIST) - set(ARG_TEST_LIST "${TARGET}_TESTS") - endif() - if(NOT ARG_DISCOVERY_TIMEOUT) - set(ARG_DISCOVERY_TIMEOUT 5) - endif() - - # Tell our test program the list of modules to scan for tests. - # We scan all modules directly listed in our SOURCES argument, and skip - # modules that came from dependencies in the DEPENDS list. - # - # This is written into a __test_modules__.py module that the test runner - # will look at. - set( - test_modules_path - "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_test_modules.py" - ) - file(WRITE "${test_modules_path}" "TEST_MODULES = [\n") - string(REPLACE "." "/" namespace_dir "${ARG_NAMESPACE}") - if (NOT "${namespace_dir}" STREQUAL "") - set(namespace_dir "${namespace_dir}/") - endif() - set(test_modules) - foreach(src_path IN LISTS ARG_SOURCES) - fb_py_compute_dest_path( - abs_source dest_path - "${src_path}" "${namespace_dir}" "${ARG_BASE_DIR}" - ) - string(REPLACE "/" "." module_name "${dest_path}") - string(REGEX REPLACE "\\.py$" "" module_name "${module_name}") - list(APPEND test_modules "${module_name}") - file(APPEND "${test_modules_path}" " '${module_name}',\n") - endforeach() - file(APPEND "${test_modules_path}" "]\n") - - # The __main__ is provided by our runner wrapper/bootstrap - list(APPEND ARG_SOURCES "${FB_PY_TEST_MAIN}=__main__.py") - list(APPEND ARG_SOURCES "${test_modules_path}=__test_modules__.py") - - if(NOT DEFINED ARG_TYPE) - set(ARG_TYPE "zipapp") - endif() - - add_fb_python_executable( - "${TARGET}" - TYPE "${ARG_TYPE}" - NAMESPACE "${ARG_NAMESPACE}" - BASE_DIR "${ARG_BASE_DIR}" - SOURCES ${ARG_SOURCES} - DEPENDS ${ARG_DEPENDS} - ) - - # Run test discovery after the test executable is built. - # This logic is based on the code for gtest_discover_tests() - set(ctest_file_base "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}") - set(ctest_include_file "${ctest_file_base}_include.cmake") - set(ctest_tests_file "${ctest_file_base}_tests.cmake") - add_custom_command( - TARGET "${TARGET}.GEN_PY_EXE" POST_BUILD - BYPRODUCTS "${ctest_tests_file}" - COMMAND - "${CMAKE_COMMAND}" - -D "TEST_TARGET=${TARGET}" - -D "TEST_INTERPRETER=${Python3_EXECUTABLE}" - -D "TEST_ENV=${ARG_ENV}" - -D "TEST_EXECUTABLE=$" - -D "TEST_WORKING_DIR=${ARG_WORKING_DIRECTORY}" - -D "TEST_LIST=${ARG_TEST_LIST}" - -D "TEST_PREFIX=${TARGET}::" - -D "TEST_PROPERTIES=${ARG_PROPERTIES}" - -D "CTEST_FILE=${ctest_tests_file}" - -P "${FB_PY_TEST_DISCOVER_SCRIPT}" - VERBATIM - ) - - file( - WRITE "${ctest_include_file}" - "if(EXISTS \"${ctest_tests_file}\")\n" - " include(\"${ctest_tests_file}\")\n" - "else()\n" - " add_test(\"${TARGET}_NOT_BUILT\" \"${TARGET}_NOT_BUILT\")\n" - "endif()\n" - ) - set_property( - DIRECTORY APPEND PROPERTY TEST_INCLUDE_FILES - "${ctest_include_file}" - ) -endfunction() - -# -# Define a python library. -# -# If you want to install a python library generated from this rule note that -# you need to use install_fb_python_library() rather than CMake's built-in -# install() function. This will make it available for other downstream -# projects to use in their add_fb_python_executable() and -# add_fb_python_library() calls. (You do still need to use `install(EXPORT)` -# later to install the CMake exports.) -# -# Parameters: -# - BASE_DIR : -# The base directory path to strip off from each source path. All source -# files must be inside this directory. If not specified it defaults to -# ${CMAKE_CURRENT_SOURCE_DIR}. -# - NAMESPACE : -# The destination namespace where these files should be installed in python -# binaries. If not specified, this defaults to the current relative path of -# ${CMAKE_CURRENT_SOURCE_DIR} inside ${CMAKE_SOURCE_DIR}. e.g., a python -# library defined in the directory repo_root/foo/bar will use a default -# namespace of "foo.bar" -# - SOURCES <...>: -# The python source files. -# You may optionally specify as source using the form: PATH=ALIAS where -# PATH is a relative path in the source tree and ALIAS is the relative -# path into which PATH should be rewritten. This is useful for mapping -# an executable script to the main module in a python executable. -# e.g.: `python/bin/watchman-wait=__main__.py` -# - DEPENDS <...>: -# Other python libraries that this one depends on. -# - INSTALL_DIR : -# The directory where this library should be installed. -# install_fb_python_library() must still be called later to perform the -# installation. If a relative path is given it will be treated relative to -# ${CMAKE_INSTALL_PREFIX} -# -# CMake is unfortunately pretty crappy at being able to define custom build -# rules & behaviors. It doesn't support transitive property propagation -# between custom targets; only the built-in add_executable() and add_library() -# targets support transitive properties. -# -# We hack around this janky CMake behavior by (ab)using interface libraries to -# propagate some of the data we want between targets, without actually -# generating a C library. -# -# add_fb_python_library(SOMELIB) generates the following things: -# - An INTERFACE library rule named SOMELIB.py_lib which tracks some -# information about transitive dependencies: -# - the transitive set of source files in the INTERFACE_SOURCES property -# - the transitive set of manifest files that this library depends on in -# the INTERFACE_INCLUDE_DIRECTORIES property. -# - A custom command that generates a SOMELIB.manifest file. -# This file contains the mapping of source files to desired destination -# locations in executables that depend on this library. This manifest file -# will then be read at build-time in order to build executables. -# -function(add_fb_python_library LIB_NAME) - fb_py_check_available() - - # Parse the arguments - # We use fb_cmake_parse_args() rather than cmake_parse_arguments() since - # cmake_parse_arguments() does not handle empty arguments, and it is common - # for callers to want to specify an empty NAMESPACE parameter. - set(one_value_args BASE_DIR NAMESPACE INSTALL_DIR) - set(multi_value_args SOURCES DEPENDS) - fb_cmake_parse_args( - ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}" - ) - fb_py_process_default_args(ARG_NAMESPACE ARG_BASE_DIR) - - string(REPLACE "." "/" namespace_dir "${ARG_NAMESPACE}") - if (NOT "${namespace_dir}" STREQUAL "") - set(namespace_dir "${namespace_dir}/") - endif() - - if(NOT DEFINED ARG_INSTALL_DIR) - set(install_dir "${FBPY_LIB_INSTALL_DIR}/") - elseif("${ARG_INSTALL_DIR}" STREQUAL "") - set(install_dir "") - else() - set(install_dir "${ARG_INSTALL_DIR}/") - endif() - - # message(STATUS "fb py library ${LIB_NAME}: " - # "NS=${namespace_dir} BASE=${ARG_BASE_DIR}") - - # TODO: In the future it would be nice to support pre-compiling the source - # files. We could emit a rule to compile each source file and emit a - # .pyc/.pyo file here, and then have the manifest reference the pyc/pyo - # files. - - # Define a library target to help pass around information about the library, - # and propagate dependency information. - # - # CMake make a lot of assumptions that libraries are C++ libraries. To help - # avoid confusion we name our target "${LIB_NAME}.py_lib" rather than just - # "${LIB_NAME}". This helps avoid confusion if callers try to use - # "${LIB_NAME}" on their own as a target name. (e.g., attempting to install - # it directly with install(TARGETS) won't work. Callers must use - # install_fb_python_library() instead.) - add_library("${LIB_NAME}.py_lib" INTERFACE) - - # Emit the manifest file. - # - # We write the manifest file to a temporary path first, then copy it with - # configure_file(COPYONLY). This is necessary to get CMake to understand - # that "${manifest_path}" is generated by the CMake configure phase, - # and allow using it as a dependency for add_custom_command(). - # (https://gitlab.kitware.com/cmake/cmake/issues/16367) - set(manifest_path "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}.manifest") - set(tmp_manifest "${manifest_path}.tmp") - file(WRITE "${tmp_manifest}" "FBPY_MANIFEST 1\n") - set(abs_sources) - foreach(src_path IN LISTS ARG_SOURCES) - fb_py_compute_dest_path( - abs_source dest_path - "${src_path}" "${namespace_dir}" "${ARG_BASE_DIR}" - ) - list(APPEND abs_sources "${abs_source}") - target_sources( - "${LIB_NAME}.py_lib" INTERFACE - "$" - "$" - ) - file( - APPEND "${tmp_manifest}" - "${abs_source} :: ${dest_path}\n" - ) - endforeach() - configure_file("${tmp_manifest}" "${manifest_path}" COPYONLY) - - target_include_directories( - "${LIB_NAME}.py_lib" INTERFACE - "$" - "$" - ) - - # Add a target that depends on all of the source files. - # This is needed in case some of the source files are generated. This will - # ensure that these source files are brought up-to-date before we build - # any python binaries that depend on this library. - add_custom_target("${LIB_NAME}.py_sources_built" DEPENDS ${abs_sources}) - add_dependencies("${LIB_NAME}.py_lib" "${LIB_NAME}.py_sources_built") - - # Hook up library dependencies, and also make the *.py_sources_built target - # depend on the sources for all of our dependencies also being up-to-date. - foreach(dep IN LISTS ARG_DEPENDS) - target_link_libraries("${LIB_NAME}.py_lib" INTERFACE "${dep}.py_lib") - - # Mark that our .py_sources_built target depends on each our our dependent - # libraries. This serves two functions: - # - This causes CMake to generate an error message if one of the - # dependencies is never defined. The target_link_libraries() call above - # won't complain if one of the dependencies doesn't exist (since it is - # intended to allow passing in file names for plain library files rather - # than just targets). - # - It ensures that sources for our dependencies are built before any - # executable that depends on us. Note that we depend on "${dep}.py_lib" - # rather than "${dep}.py_sources_built" for this purpose because the - # ".py_sources_built" target won't be available for imported targets. - add_dependencies("${LIB_NAME}.py_sources_built" "${dep}.py_lib") - endforeach() - - # Add a custom command to help with library installation, in case - # install_fb_python_library() is called later for this library. - # add_custom_command() only works with file dependencies defined in the same - # CMakeLists.txt file, so we want to make sure this is defined here, rather - # then where install_fb_python_library() is called. - # This command won't be run by default, but will only be run if it is needed - # by a subsequent install_fb_python_library() call. - # - # This command copies the library contents into the build directory. - # It would be nicer if we could skip this intermediate copy, and just run - # make_fbpy_archive.py at install time to copy them directly to the desired - # installation directory. Unfortunately this is difficult to do, and seems - # to interfere with some of the CMake code that wants to generate a manifest - # of installed files. - set(build_install_dir "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}.lib_install") - add_custom_command( - OUTPUT - "${build_install_dir}/${LIB_NAME}.manifest" - COMMAND "${CMAKE_COMMAND}" -E remove_directory "${build_install_dir}" - COMMAND - "${Python3_EXECUTABLE}" "${FB_MAKE_PYTHON_ARCHIVE}" --type lib-install - --install-dir "${LIB_NAME}" - -o "${build_install_dir}/${LIB_NAME}" "${manifest_path}" - DEPENDS - "${abs_sources}" - "${manifest_path}" - "${FB_MAKE_PYTHON_ARCHIVE}" - ) - add_custom_target( - "${LIB_NAME}.py_lib_install" - DEPENDS "${build_install_dir}/${LIB_NAME}.manifest" - ) - - # Set some properties to pass through the install paths to - # install_fb_python_library() - # - # Passing through ${build_install_dir} allows install_fb_python_library() - # to work even if used from a different CMakeLists.txt file than where - # add_fb_python_library() was called (i.e. such that - # ${CMAKE_CURRENT_BINARY_DIR} is different between the two calls). - set(abs_install_dir "${install_dir}") - if(NOT IS_ABSOLUTE "${abs_install_dir}") - set(abs_install_dir "${CMAKE_INSTALL_PREFIX}/${abs_install_dir}") - endif() - string(REGEX REPLACE "/$" "" abs_install_dir "${abs_install_dir}") - set_target_properties( - "${LIB_NAME}.py_lib_install" - PROPERTIES - INSTALL_DIR "${abs_install_dir}" - BUILD_INSTALL_DIR "${build_install_dir}" - ) -endfunction() - -# -# Install an FB-style packaged python binary. -# -# - DESTINATION : -# Associate the installed target files with the given export-name. -# -function(install_fb_python_executable TARGET) - # Parse the arguments - set(one_value_args DESTINATION) - set(multi_value_args) - fb_cmake_parse_args( - ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}" - ) - - if(NOT DEFINED ARG_DESTINATION) - set(ARG_DESTINATION bin) - endif() - - install( - PROGRAMS "$" - DESTINATION "${ARG_DESTINATION}" - ) -endfunction() - -# -# Install a python library. -# -# - EXPORT : -# Associate the installed target files with the given export-name. -# -# Note that unlike the built-in CMake install() function we do not accept a -# DESTINATION parameter. Instead, use the INSTALL_DIR parameter to -# add_fb_python_library() to set the installation location. -# -function(install_fb_python_library LIB_NAME) - set(one_value_args EXPORT) - fb_cmake_parse_args(ARG "" "${one_value_args}" "" "${ARGN}") - - # Export our "${LIB_NAME}.py_lib" target so that it will be available to - # downstream projects in our installed CMake config files. - if(DEFINED ARG_EXPORT) - install(TARGETS "${LIB_NAME}.py_lib" EXPORT "${ARG_EXPORT}") - endif() - - # add_fb_python_library() emits a .py_lib_install target that will prepare - # the installation directory. However, it isn't part of the "ALL" target and - # therefore isn't built by default. - # - # Make sure the ALL target depends on it now. We have to do this by - # introducing yet another custom target. - # Add it as a dependency to the ALL target now. - add_custom_target("${LIB_NAME}.py_lib_install_all" ALL) - add_dependencies( - "${LIB_NAME}.py_lib_install_all" "${LIB_NAME}.py_lib_install" - ) - - # Copy the intermediate install directory generated at build time into - # the desired install location. - get_target_property(dest_dir "${LIB_NAME}.py_lib_install" "INSTALL_DIR") - get_target_property( - build_install_dir "${LIB_NAME}.py_lib_install" "BUILD_INSTALL_DIR" - ) - install( - DIRECTORY "${build_install_dir}/${LIB_NAME}" - DESTINATION "${dest_dir}" - ) - install( - FILES "${build_install_dir}/${LIB_NAME}.manifest" - DESTINATION "${dest_dir}" - ) -endfunction() - -# Helper macro to process the BASE_DIR and NAMESPACE arguments for -# add_fb_python_executable() and add_fb_python_executable() -macro(fb_py_process_default_args NAMESPACE_VAR BASE_DIR_VAR) - # If the namespace was not specified, default to the relative path to the - # current directory (starting from the repository root). - if(NOT DEFINED "${NAMESPACE_VAR}") - file( - RELATIVE_PATH "${NAMESPACE_VAR}" - "${CMAKE_SOURCE_DIR}" - "${CMAKE_CURRENT_SOURCE_DIR}" - ) - endif() - - if(NOT DEFINED "${BASE_DIR_VAR}") - # If the base directory was not specified, default to the current directory - set("${BASE_DIR_VAR}" "${CMAKE_CURRENT_SOURCE_DIR}") - else() - # If the base directory was specified, always convert it to an - # absolute path. - get_filename_component("${BASE_DIR_VAR}" "${${BASE_DIR_VAR}}" ABSOLUTE) - endif() -endmacro() - -function(fb_py_check_available) - # Make sure that Python 3 and our make_fbpy_archive.py helper script are - # available. - if(NOT Python3_EXECUTABLE) - if(FBPY_FIND_PYTHON_ERR) - message(FATAL_ERROR "Unable to find Python 3: ${FBPY_FIND_PYTHON_ERR}") - else() - message(FATAL_ERROR "Unable to find Python 3") - endif() - endif() - - if (NOT FB_MAKE_PYTHON_ARCHIVE) - message( - FATAL_ERROR "unable to find make_fbpy_archive.py helper program (it " - "should be located in the same directory as FBPythonBinary.cmake)" - ) - endif() -endfunction() - -function( - fb_py_compute_dest_path - src_path_output dest_path_output src_path namespace_dir base_dir -) - if("${src_path}" MATCHES "=") - # We want to split the string on the `=` sign, but cmake doesn't - # provide much in the way of helpers for this, so we rewrite the - # `=` sign to `;` so that we can treat it as a cmake list and - # then index into the components - string(REPLACE "=" ";" src_path_list "${src_path}") - list(GET src_path_list 0 src_path) - # Note that we ignore the `namespace_dir` in the alias case - # in order to allow aliasing a source to the top level `__main__.py` - # filename. - list(GET src_path_list 1 dest_path) - else() - unset(dest_path) - endif() - - get_filename_component(abs_source "${src_path}" ABSOLUTE) - if(NOT DEFINED dest_path) - file(RELATIVE_PATH rel_src "${ARG_BASE_DIR}" "${abs_source}") - if("${rel_src}" MATCHES "^../") - message( - FATAL_ERROR "${LIB_NAME}: source file \"${abs_source}\" is not inside " - "the base directory ${ARG_BASE_DIR}" - ) - endif() - set(dest_path "${namespace_dir}${rel_src}") - endif() - - set("${src_path_output}" "${abs_source}" PARENT_SCOPE) - set("${dest_path_output}" "${dest_path}" PARENT_SCOPE) -endfunction() diff --git a/build/fbcode_builder/CMake/FBPythonTestAddTests.cmake b/build/fbcode_builder/CMake/FBPythonTestAddTests.cmake deleted file mode 100644 index d73c055d8..000000000 --- a/build/fbcode_builder/CMake/FBPythonTestAddTests.cmake +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. - -# Add a command to be emitted to the CTest file -set(ctest_script) -function(add_command CMD) - set(escaped_args "") - foreach(arg ${ARGN}) - # Escape all arguments using "Bracket Argument" syntax - # We could skip this for argument that don't contain any special - # characters if we wanted to make the output slightly more human-friendly. - set(escaped_args "${escaped_args} [==[${arg}]==]") - endforeach() - set(ctest_script "${ctest_script}${CMD}(${escaped_args})\n" PARENT_SCOPE) -endfunction() - -if(NOT EXISTS "${TEST_EXECUTABLE}") - message(FATAL_ERROR "Test executable does not exist: ${TEST_EXECUTABLE}") -endif() -execute_process( - COMMAND ${CMAKE_COMMAND} -E env ${TEST_ENV} "${TEST_INTERPRETER}" "${TEST_EXECUTABLE}" --list-tests - WORKING_DIRECTORY "${TEST_WORKING_DIR}" - OUTPUT_VARIABLE output - RESULT_VARIABLE result -) -if(NOT "${result}" EQUAL 0) - string(REPLACE "\n" "\n " output "${output}") - message( - FATAL_ERROR - "Error running test executable: ${TEST_EXECUTABLE}\n" - "Output:\n" - " ${output}\n" - ) -endif() - -# Parse output -string(REPLACE "\n" ";" tests_list "${output}") -foreach(test_name ${tests_list}) - add_command( - add_test - "${TEST_PREFIX}${test_name}" - ${CMAKE_COMMAND} -E env ${TEST_ENV} - "${TEST_INTERPRETER}" "${TEST_EXECUTABLE}" "${test_name}" - ) - add_command( - set_tests_properties - "${TEST_PREFIX}${test_name}" - PROPERTIES - WORKING_DIRECTORY "${TEST_WORKING_DIR}" - ${TEST_PROPERTIES} - ) -endforeach() - -# Set a list of discovered tests in the parent scope, in case users -# want access to this list as a CMake variable -if(TEST_LIST) - add_command(set ${TEST_LIST} ${tests_list}) -endif() - -file(WRITE "${CTEST_FILE}" "${ctest_script}") diff --git a/build/fbcode_builder/CMake/FBThriftCppLibrary.cmake b/build/fbcode_builder/CMake/FBThriftCppLibrary.cmake deleted file mode 100644 index 416a88b75..000000000 --- a/build/fbcode_builder/CMake/FBThriftCppLibrary.cmake +++ /dev/null @@ -1,202 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. - -include(FBCMakeParseArgs) - -# Generate a C++ library from a thrift file -# -# Parameters: -# - SERVICES [ ...] -# The names of the services defined in the thrift file. -# - DEPENDS [ ...] -# A list of other thrift C++ libraries that this library depends on. -# - OPTIONS [ ...] -# A list of options to pass to the thrift compiler. -# - INCLUDE_DIR -# The sub-directory where generated headers will be installed. -# Defaults to "include" if not specified. The caller must still call -# install() to install the thrift library if desired. -# - THRIFT_INCLUDE_DIR -# The sub-directory where generated headers will be installed. -# Defaults to "${INCLUDE_DIR}/thrift-files" if not specified. -# The caller must still call install() to install the thrift library if -# desired. -function(add_fbthrift_cpp_library LIB_NAME THRIFT_FILE) - # Parse the arguments - set(one_value_args INCLUDE_DIR THRIFT_INCLUDE_DIR) - set(multi_value_args SERVICES DEPENDS OPTIONS) - fb_cmake_parse_args( - ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}" - ) - if(NOT DEFINED ARG_INCLUDE_DIR) - set(ARG_INCLUDE_DIR "include") - endif() - if(NOT DEFINED ARG_THRIFT_INCLUDE_DIR) - set(ARG_THRIFT_INCLUDE_DIR "${ARG_INCLUDE_DIR}/thrift-files") - endif() - - get_filename_component(base ${THRIFT_FILE} NAME_WE) - get_filename_component( - output_dir - ${CMAKE_CURRENT_BINARY_DIR}/${THRIFT_FILE} - DIRECTORY - ) - - # Generate relative paths in #includes - file( - RELATIVE_PATH include_prefix - "${CMAKE_SOURCE_DIR}" - "${CMAKE_CURRENT_SOURCE_DIR}/${THRIFT_FILE}" - ) - get_filename_component(include_prefix ${include_prefix} DIRECTORY) - - if (NOT "${include_prefix}" STREQUAL "") - list(APPEND ARG_OPTIONS "include_prefix=${include_prefix}") - endif() - # CMake 3.12 is finally getting a list(JOIN) function, but until then - # treating the list as a string and replacing the semicolons is good enough. - string(REPLACE ";" "," GEN_ARG_STR "${ARG_OPTIONS}") - - # Compute the list of generated files - list(APPEND generated_headers - "${output_dir}/gen-cpp2/${base}_constants.h" - "${output_dir}/gen-cpp2/${base}_types.h" - "${output_dir}/gen-cpp2/${base}_types.tcc" - "${output_dir}/gen-cpp2/${base}_types_custom_protocol.h" - "${output_dir}/gen-cpp2/${base}_metadata.h" - ) - list(APPEND generated_sources - "${output_dir}/gen-cpp2/${base}_constants.cpp" - "${output_dir}/gen-cpp2/${base}_data.h" - "${output_dir}/gen-cpp2/${base}_data.cpp" - "${output_dir}/gen-cpp2/${base}_types.cpp" - "${output_dir}/gen-cpp2/${base}_types_binary.cpp" - "${output_dir}/gen-cpp2/${base}_types_compact.cpp" - "${output_dir}/gen-cpp2/${base}_types_serialization.cpp" - "${output_dir}/gen-cpp2/${base}_metadata.cpp" - ) - foreach(service IN LISTS ARG_SERVICES) - list(APPEND generated_headers - "${output_dir}/gen-cpp2/${service}.h" - "${output_dir}/gen-cpp2/${service}.tcc" - "${output_dir}/gen-cpp2/${service}AsyncClient.h" - "${output_dir}/gen-cpp2/${service}_custom_protocol.h" - ) - list(APPEND generated_sources - "${output_dir}/gen-cpp2/${service}.cpp" - "${output_dir}/gen-cpp2/${service}AsyncClient.cpp" - "${output_dir}/gen-cpp2/${service}_processmap_binary.cpp" - "${output_dir}/gen-cpp2/${service}_processmap_compact.cpp" - ) - endforeach() - - # This generator expression gets the list of include directories required - # for all of our dependencies. - # It requires using COMMAND_EXPAND_LISTS in the add_custom_command() call - # below. COMMAND_EXPAND_LISTS is only available in CMake 3.8+ - # If we really had to support older versions of CMake we would probably need - # to use a wrapper script around the thrift compiler that could take the - # include list as a single argument and split it up before invoking the - # thrift compiler. - if (NOT POLICY CMP0067) - message(FATAL_ERROR "add_fbthrift_cpp_library() requires CMake 3.8+") - endif() - set( - thrift_include_options - "-I;$,;-I;>" - ) - - # Emit the rule to run the thrift compiler - add_custom_command( - OUTPUT - ${generated_headers} - ${generated_sources} - COMMAND_EXPAND_LISTS - COMMAND - "${CMAKE_COMMAND}" -E make_directory "${output_dir}" - COMMAND - "${FBTHRIFT_COMPILER}" - --legacy-strict - --gen "mstch_cpp2:${GEN_ARG_STR}" - "${thrift_include_options}" - -I "${FBTHRIFT_INCLUDE_DIR}" - -o "${output_dir}" - "${CMAKE_CURRENT_SOURCE_DIR}/${THRIFT_FILE}" - WORKING_DIRECTORY - "${CMAKE_BINARY_DIR}" - MAIN_DEPENDENCY - "${THRIFT_FILE}" - DEPENDS - ${ARG_DEPENDS} - "${FBTHRIFT_COMPILER}" - ) - - # Now emit the library rule to compile the sources - if (BUILD_SHARED_LIBS) - set(LIB_TYPE SHARED) - else () - set(LIB_TYPE STATIC) - endif () - - add_library( - "${LIB_NAME}" ${LIB_TYPE} - ${generated_sources} - ) - - target_include_directories( - "${LIB_NAME}" - PUBLIC - "$" - "$" - ${Xxhash_INCLUDE_DIR} - ) - target_link_libraries( - "${LIB_NAME}" - PUBLIC - ${ARG_DEPENDS} - FBThrift::thriftcpp2 - Folly::folly - mvfst::mvfst_server_async_tran - mvfst::mvfst_server - ${Xxhash_LIBRARY} - ) - - # Add ${generated_headers} to the PUBLIC_HEADER property for ${LIB_NAME} - # - # This allows callers to install it using - # "install(TARGETS ${LIB_NAME} PUBLIC_HEADER)" - # However, note that CMake's PUBLIC_HEADER behavior is rather inflexible, - # and does have any way to preserve header directory structure. Callers - # must be careful to use the correct PUBLIC_HEADER DESTINATION parameter - # when doing this, to put the files the correct directory themselves. - # We define a HEADER_INSTALL_DIR property with the include directory prefix, - # so typically callers should specify the PUBLIC_HEADER DESTINATION as - # "$" - set_property( - TARGET "${LIB_NAME}" - PROPERTY PUBLIC_HEADER ${generated_headers} - ) - - # Define a dummy interface library to help propagate the thrift include - # directories between dependencies. - add_library("${LIB_NAME}.thrift_includes" INTERFACE) - target_include_directories( - "${LIB_NAME}.thrift_includes" - INTERFACE - "$" - "$" - ) - foreach(dep IN LISTS ARG_DEPENDS) - target_link_libraries( - "${LIB_NAME}.thrift_includes" - INTERFACE "${dep}.thrift_includes" - ) - endforeach() - - set_target_properties( - "${LIB_NAME}" - PROPERTIES - EXPORT_PROPERTIES "THRIFT_INSTALL_DIR" - THRIFT_INSTALL_DIR "${ARG_THRIFT_INCLUDE_DIR}/${include_prefix}" - HEADER_INSTALL_DIR "${ARG_INCLUDE_DIR}/${include_prefix}/gen-cpp2" - ) -endfunction() diff --git a/build/fbcode_builder/CMake/FBThriftLibrary.cmake b/build/fbcode_builder/CMake/FBThriftLibrary.cmake deleted file mode 100644 index e4280e2a4..000000000 --- a/build/fbcode_builder/CMake/FBThriftLibrary.cmake +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. - -include(FBCMakeParseArgs) -include(FBThriftPyLibrary) -include(FBThriftCppLibrary) - -# -# add_fbthrift_library() -# -# This is a convenience function that generates thrift libraries for multiple -# languages. -# -# For example: -# add_fbthrift_library( -# foo foo.thrift -# LANGUAGES cpp py -# SERVICES Foo -# DEPENDS bar) -# -# will be expanded into two separate calls: -# -# add_fbthrift_cpp_library(foo_cpp foo.thrift SERVICES Foo DEPENDS bar_cpp) -# add_fbthrift_py_library(foo_py foo.thrift SERVICES Foo DEPENDS bar_py) -# -function(add_fbthrift_library LIB_NAME THRIFT_FILE) - # Parse the arguments - set(one_value_args PY_NAMESPACE INCLUDE_DIR THRIFT_INCLUDE_DIR) - set(multi_value_args SERVICES DEPENDS LANGUAGES CPP_OPTIONS PY_OPTIONS) - fb_cmake_parse_args( - ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}" - ) - - if(NOT DEFINED ARG_INCLUDE_DIR) - set(ARG_INCLUDE_DIR "include") - endif() - if(NOT DEFINED ARG_THRIFT_INCLUDE_DIR) - set(ARG_THRIFT_INCLUDE_DIR "${ARG_INCLUDE_DIR}/thrift-files") - endif() - - # CMake 3.12+ adds list(TRANSFORM) which would be nice to use here, but for - # now we still want to support older versions of CMake. - set(CPP_DEPENDS) - set(PY_DEPENDS) - foreach(dep IN LISTS ARG_DEPENDS) - list(APPEND CPP_DEPENDS "${dep}_cpp") - list(APPEND PY_DEPENDS "${dep}_py") - endforeach() - - foreach(lang IN LISTS ARG_LANGUAGES) - if ("${lang}" STREQUAL "cpp") - add_fbthrift_cpp_library( - "${LIB_NAME}_cpp" "${THRIFT_FILE}" - SERVICES ${ARG_SERVICES} - DEPENDS ${CPP_DEPENDS} - OPTIONS ${ARG_CPP_OPTIONS} - INCLUDE_DIR "${ARG_INCLUDE_DIR}" - THRIFT_INCLUDE_DIR "${ARG_THRIFT_INCLUDE_DIR}" - ) - elseif ("${lang}" STREQUAL "py" OR "${lang}" STREQUAL "python") - if (DEFINED ARG_PY_NAMESPACE) - set(namespace_args NAMESPACE "${ARG_PY_NAMESPACE}") - endif() - add_fbthrift_py_library( - "${LIB_NAME}_py" "${THRIFT_FILE}" - SERVICES ${ARG_SERVICES} - ${namespace_args} - DEPENDS ${PY_DEPENDS} - OPTIONS ${ARG_PY_OPTIONS} - THRIFT_INCLUDE_DIR "${ARG_THRIFT_INCLUDE_DIR}" - ) - else() - message( - FATAL_ERROR "unknown language for thrift library ${LIB_NAME}: ${lang}" - ) - endif() - endforeach() -endfunction() diff --git a/build/fbcode_builder/CMake/FBThriftPyLibrary.cmake b/build/fbcode_builder/CMake/FBThriftPyLibrary.cmake deleted file mode 100644 index fa77cde71..000000000 --- a/build/fbcode_builder/CMake/FBThriftPyLibrary.cmake +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. - -include(FBCMakeParseArgs) -include(FBPythonBinary) - -# Generate a Python library from a thrift file -function(add_fbthrift_py_library LIB_NAME THRIFT_FILE) - # Parse the arguments - set(one_value_args NAMESPACE THRIFT_INCLUDE_DIR) - set(multi_value_args SERVICES DEPENDS OPTIONS) - fb_cmake_parse_args( - ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}" - ) - - if(NOT DEFINED ARG_THRIFT_INCLUDE_DIR) - set(ARG_THRIFT_INCLUDE_DIR "include/thrift-files") - endif() - - get_filename_component(base ${THRIFT_FILE} NAME_WE) - set(output_dir "${CMAKE_CURRENT_BINARY_DIR}/${THRIFT_FILE}-py") - - # Parse the namespace value - if (NOT DEFINED ARG_NAMESPACE) - set(ARG_NAMESPACE "${base}") - endif() - - string(REPLACE "." "/" namespace_dir "${ARG_NAMESPACE}") - set(py_output_dir "${output_dir}/gen-py/${namespace_dir}") - list(APPEND generated_sources - "${py_output_dir}/__init__.py" - "${py_output_dir}/ttypes.py" - "${py_output_dir}/constants.py" - ) - foreach(service IN LISTS ARG_SERVICES) - list(APPEND generated_sources - ${py_output_dir}/${service}.py - ) - endforeach() - - # Define a dummy interface library to help propagate the thrift include - # directories between dependencies. - add_library("${LIB_NAME}.thrift_includes" INTERFACE) - target_include_directories( - "${LIB_NAME}.thrift_includes" - INTERFACE - "$" - "$" - ) - foreach(dep IN LISTS ARG_DEPENDS) - target_link_libraries( - "${LIB_NAME}.thrift_includes" - INTERFACE "${dep}.thrift_includes" - ) - endforeach() - - # This generator expression gets the list of include directories required - # for all of our dependencies. - # It requires using COMMAND_EXPAND_LISTS in the add_custom_command() call - # below. COMMAND_EXPAND_LISTS is only available in CMake 3.8+ - # If we really had to support older versions of CMake we would probably need - # to use a wrapper script around the thrift compiler that could take the - # include list as a single argument and split it up before invoking the - # thrift compiler. - if (NOT POLICY CMP0067) - message(FATAL_ERROR "add_fbthrift_py_library() requires CMake 3.8+") - endif() - set( - thrift_include_options - "-I;$,;-I;>" - ) - - # Always force generation of "new-style" python classes for Python 2 - list(APPEND ARG_OPTIONS "new_style") - # CMake 3.12 is finally getting a list(JOIN) function, but until then - # treating the list as a string and replacing the semicolons is good enough. - string(REPLACE ";" "," GEN_ARG_STR "${ARG_OPTIONS}") - - # Emit the rule to run the thrift compiler - add_custom_command( - OUTPUT - ${generated_sources} - COMMAND_EXPAND_LISTS - COMMAND - "${CMAKE_COMMAND}" -E make_directory "${output_dir}" - COMMAND - "${FBTHRIFT_COMPILER}" - --legacy-strict - --gen "py:${GEN_ARG_STR}" - "${thrift_include_options}" - -o "${output_dir}" - "${CMAKE_CURRENT_SOURCE_DIR}/${THRIFT_FILE}" - WORKING_DIRECTORY - "${CMAKE_BINARY_DIR}" - MAIN_DEPENDENCY - "${THRIFT_FILE}" - DEPENDS - "${FBTHRIFT_COMPILER}" - ) - - # We always want to pass the namespace as "" to this call: - # thrift will already emit the files with the desired namespace prefix under - # gen-py. We don't want add_fb_python_library() to prepend the namespace a - # second time. - add_fb_python_library( - "${LIB_NAME}" - BASE_DIR "${output_dir}/gen-py" - NAMESPACE "" - SOURCES ${generated_sources} - DEPENDS ${ARG_DEPENDS} FBThrift::thrift_py - ) -endfunction() diff --git a/build/fbcode_builder/CMake/FindCares.cmake b/build/fbcode_builder/CMake/FindCares.cmake deleted file mode 100644 index d0d4ca583..000000000 --- a/build/fbcode_builder/CMake/FindCares.cmake +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -find_path(CARES_INCLUDE_DIR NAMES ares.h) -find_library(CARES_LIBRARIES NAMES cares) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(Cares DEFAULT_MSG CARES_LIBRARIES CARES_INCLUDE_DIR) - -mark_as_advanced( - CARES_LIBRARIES - CARES_INCLUDE_DIR -) - -if(NOT TARGET cares) - if("${CARES_LIBRARIES}" MATCHES ".*.a$") - add_library(cares STATIC IMPORTED) - else() - add_library(cares SHARED IMPORTED) - endif() - set_target_properties( - cares - PROPERTIES - IMPORTED_LOCATION ${CARES_LIBRARIES} - INTERFACE_INCLUDE_DIRECTORIES ${CARES_INCLUDE_DIR} - ) -endif() diff --git a/build/fbcode_builder/CMake/FindDoubleConversion.cmake b/build/fbcode_builder/CMake/FindDoubleConversion.cmake deleted file mode 100644 index 12a423bc1..000000000 --- a/build/fbcode_builder/CMake/FindDoubleConversion.cmake +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. - -# Finds libdouble-conversion. -# -# This module defines: -# DOUBLE_CONVERSION_INCLUDE_DIR -# DOUBLE_CONVERSION_LIBRARY -# - -find_path(DOUBLE_CONVERSION_INCLUDE_DIR double-conversion/double-conversion.h) -find_library(DOUBLE_CONVERSION_LIBRARY NAMES double-conversion) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args( - DoubleConversion - DEFAULT_MSG - DOUBLE_CONVERSION_LIBRARY DOUBLE_CONVERSION_INCLUDE_DIR) - -mark_as_advanced(DOUBLE_CONVERSION_INCLUDE_DIR DOUBLE_CONVERSION_LIBRARY) diff --git a/build/fbcode_builder/CMake/FindGMock.cmake b/build/fbcode_builder/CMake/FindGMock.cmake deleted file mode 100644 index cd042dd9c..000000000 --- a/build/fbcode_builder/CMake/FindGMock.cmake +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# Find libgmock -# -# LIBGMOCK_DEFINES - List of defines when using libgmock. -# LIBGMOCK_INCLUDE_DIR - where to find gmock/gmock.h, etc. -# LIBGMOCK_LIBRARIES - List of libraries when using libgmock. -# LIBGMOCK_FOUND - True if libgmock found. - -IF (LIBGMOCK_INCLUDE_DIR) - # Already in cache, be silent - SET(LIBGMOCK_FIND_QUIETLY TRUE) -ENDIF () - -find_package(GTest CONFIG QUIET) -if (TARGET GTest::gmock) - get_target_property(LIBGMOCK_DEFINES GTest::gtest INTERFACE_COMPILE_DEFINITIONS) - if (NOT ${LIBGMOCK_DEFINES}) - # Explicitly set to empty string if not found to avoid it being - # set to NOTFOUND and breaking compilation - set(LIBGMOCK_DEFINES "") - endif() - get_target_property(LIBGMOCK_INCLUDE_DIR GTest::gtest INTERFACE_INCLUDE_DIRECTORIES) - set(LIBGMOCK_LIBRARIES GTest::gmock_main GTest::gmock GTest::gtest) - set(LIBGMOCK_FOUND ON) - message(STATUS "Found gmock via config, defines=${LIBGMOCK_DEFINES}, include=${LIBGMOCK_INCLUDE_DIR}, libs=${LIBGMOCK_LIBRARIES}") -else() - - FIND_PATH(LIBGMOCK_INCLUDE_DIR gmock/gmock.h) - - FIND_LIBRARY(LIBGMOCK_MAIN_LIBRARY_DEBUG NAMES gmock_maind) - FIND_LIBRARY(LIBGMOCK_MAIN_LIBRARY_RELEASE NAMES gmock_main) - FIND_LIBRARY(LIBGMOCK_LIBRARY_DEBUG NAMES gmockd) - FIND_LIBRARY(LIBGMOCK_LIBRARY_RELEASE NAMES gmock) - FIND_LIBRARY(LIBGTEST_LIBRARY_DEBUG NAMES gtestd) - FIND_LIBRARY(LIBGTEST_LIBRARY_RELEASE NAMES gtest) - - find_package(Threads REQUIRED) - INCLUDE(SelectLibraryConfigurations) - SELECT_LIBRARY_CONFIGURATIONS(LIBGMOCK_MAIN) - SELECT_LIBRARY_CONFIGURATIONS(LIBGMOCK) - SELECT_LIBRARY_CONFIGURATIONS(LIBGTEST) - - set(LIBGMOCK_LIBRARIES - ${LIBGMOCK_MAIN_LIBRARY} - ${LIBGMOCK_LIBRARY} - ${LIBGTEST_LIBRARY} - Threads::Threads - ) - - if(CMAKE_SYSTEM_NAME STREQUAL "Windows") - # The GTEST_LINKED_AS_SHARED_LIBRARY macro must be set properly on Windows. - # - # There isn't currently an easy way to determine if a library was compiled as - # a shared library on Windows, so just assume we've been built against a - # shared build of gmock for now. - SET(LIBGMOCK_DEFINES "GTEST_LINKED_AS_SHARED_LIBRARY=1" CACHE STRING "") - endif() - - # handle the QUIETLY and REQUIRED arguments and set LIBGMOCK_FOUND to TRUE if - # all listed variables are TRUE - INCLUDE(FindPackageHandleStandardArgs) - FIND_PACKAGE_HANDLE_STANDARD_ARGS( - GMock - DEFAULT_MSG - LIBGMOCK_MAIN_LIBRARY - LIBGMOCK_LIBRARY - LIBGTEST_LIBRARY - LIBGMOCK_LIBRARIES - LIBGMOCK_INCLUDE_DIR - ) - - MARK_AS_ADVANCED( - LIBGMOCK_DEFINES - LIBGMOCK_MAIN_LIBRARY - LIBGMOCK_LIBRARY - LIBGTEST_LIBRARY - LIBGMOCK_LIBRARIES - LIBGMOCK_INCLUDE_DIR - ) -endif() diff --git a/build/fbcode_builder/CMake/FindGflags.cmake b/build/fbcode_builder/CMake/FindGflags.cmake deleted file mode 100644 index 0101203e0..000000000 --- a/build/fbcode_builder/CMake/FindGflags.cmake +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# Find libgflags. -# There's a lot of compatibility cruft going on in here, both -# to deal with changes across the FB consumers of this and also -# to deal with variances in behavior of cmake itself. -# -# Since this file is named FindGflags.cmake the cmake convention -# is for the module to export both GFLAGS_FOUND and Gflags_FOUND. -# The convention expected by consumers is that we export the -# following variables, even though these do not match the cmake -# conventions: -# -# LIBGFLAGS_INCLUDE_DIR - where to find gflags/gflags.h, etc. -# LIBGFLAGS_LIBRARY - List of libraries when using libgflags. -# LIBGFLAGS_FOUND - True if libgflags found. -# -# We need to be able to locate gflags both from an installed -# cmake config file and just from the raw headers and libs, so -# test for the former and then the latter, and then stick -# the results together and export them into the variables -# listed above. -# -# For forwards compatibility, we export the following variables: -# -# gflags_INCLUDE_DIR - where to find gflags/gflags.h, etc. -# gflags_TARGET / GFLAGS_TARGET / gflags_LIBRARIES -# - List of libraries when using libgflags. -# gflags_FOUND - True if libgflags found. -# - -IF (LIBGFLAGS_INCLUDE_DIR) - # Already in cache, be silent - SET(Gflags_FIND_QUIETLY TRUE) -ENDIF () - -find_package(gflags CONFIG QUIET) -if (gflags_FOUND) - if (NOT Gflags_FIND_QUIETLY) - message(STATUS "Found gflags from package config ${gflags_CONFIG}") - endif() - # Re-export the config-specified libs with our local names - set(LIBGFLAGS_LIBRARY ${gflags_LIBRARIES}) - set(LIBGFLAGS_INCLUDE_DIR ${gflags_INCLUDE_DIR}) - if(NOT EXISTS "${gflags_INCLUDE_DIR}") - # The gflags-devel RPM on recent RedHat-based systems is somewhat broken. - # RedHat symlinks /lib64 to /usr/lib64, and this breaks some of the - # relative path computation performed in gflags-config.cmake. The package - # config file ends up being found via /lib64, but the relative path - # computation it does only works if it was found in /usr/lib64. - # If gflags_INCLUDE_DIR does not actually exist, simply default it to - # /usr/include on these systems. - set(LIBGFLAGS_INCLUDE_DIR "/usr/include") - set(GFLAGS_INCLUDE_DIR "/usr/include") - endif() - set(LIBGFLAGS_FOUND ${gflags_FOUND}) - # cmake module compat - set(GFLAGS_FOUND ${gflags_FOUND}) - set(Gflags_FOUND ${gflags_FOUND}) -else() - FIND_PATH(LIBGFLAGS_INCLUDE_DIR gflags/gflags.h) - - FIND_LIBRARY(LIBGFLAGS_LIBRARY_DEBUG NAMES gflagsd gflags_staticd) - FIND_LIBRARY(LIBGFLAGS_LIBRARY_RELEASE NAMES gflags gflags_static) - - INCLUDE(SelectLibraryConfigurations) - SELECT_LIBRARY_CONFIGURATIONS(LIBGFLAGS) - - # handle the QUIETLY and REQUIRED arguments and set LIBGFLAGS_FOUND to TRUE if - # all listed variables are TRUE - INCLUDE(FindPackageHandleStandardArgs) - FIND_PACKAGE_HANDLE_STANDARD_ARGS(gflags DEFAULT_MSG LIBGFLAGS_LIBRARY LIBGFLAGS_INCLUDE_DIR) - # cmake module compat - set(Gflags_FOUND ${GFLAGS_FOUND}) - # compat with some existing FindGflags consumers - set(LIBGFLAGS_FOUND ${GFLAGS_FOUND}) - - # Compat with the gflags CONFIG based detection - set(gflags_FOUND ${GFLAGS_FOUND}) - set(gflags_INCLUDE_DIR ${LIBGFLAGS_INCLUDE_DIR}) - set(gflags_LIBRARIES ${LIBGFLAGS_LIBRARY}) - set(GFLAGS_TARGET ${LIBGFLAGS_LIBRARY}) - set(gflags_TARGET ${LIBGFLAGS_LIBRARY}) - - MARK_AS_ADVANCED(LIBGFLAGS_LIBRARY LIBGFLAGS_INCLUDE_DIR) -endif() - -# Compat with the gflags CONFIG based detection -if (LIBGFLAGS_FOUND AND NOT TARGET gflags) - add_library(gflags UNKNOWN IMPORTED) - if(TARGET gflags-shared) - # If the installed gflags CMake package config defines a gflags-shared - # target but not gflags, just make the gflags target that we define - # depend on the gflags-shared target. - target_link_libraries(gflags INTERFACE gflags-shared) - # Export LIBGFLAGS_LIBRARY as the gflags-shared target in this case. - set(LIBGFLAGS_LIBRARY gflags-shared) - else() - set_target_properties( - gflags - PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES "C" - IMPORTED_LOCATION "${LIBGFLAGS_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${LIBGFLAGS_INCLUDE_DIR}" - ) - endif() -endif() diff --git a/build/fbcode_builder/CMake/FindGlog.cmake b/build/fbcode_builder/CMake/FindGlog.cmake deleted file mode 100644 index e8b277a38..000000000 --- a/build/fbcode_builder/CMake/FindGlog.cmake +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# - Try to find Glog -# Once done, this will define -# -# GLOG_FOUND - system has Glog -# GLOG_INCLUDE_DIRS - the Glog include directories -# GLOG_LIBRARIES - link these to use Glog - -include(FindPackageHandleStandardArgs) -include(SelectLibraryConfigurations) - -find_library(GLOG_LIBRARY_RELEASE glog - PATHS ${GLOG_LIBRARYDIR}) -find_library(GLOG_LIBRARY_DEBUG glogd - PATHS ${GLOG_LIBRARYDIR}) - -find_path(GLOG_INCLUDE_DIR glog/logging.h - PATHS ${GLOG_INCLUDEDIR}) - -select_library_configurations(GLOG) - -find_package_handle_standard_args(Glog DEFAULT_MSG - GLOG_LIBRARY - GLOG_INCLUDE_DIR) - -mark_as_advanced( - GLOG_LIBRARY - GLOG_INCLUDE_DIR) - -set(GLOG_LIBRARIES ${GLOG_LIBRARY}) -set(GLOG_INCLUDE_DIRS ${GLOG_INCLUDE_DIR}) - -if (NOT TARGET glog::glog) - add_library(glog::glog UNKNOWN IMPORTED) - set_target_properties(glog::glog PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${GLOG_INCLUDE_DIRS}") - set_target_properties(glog::glog PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C" IMPORTED_LOCATION "${GLOG_LIBRARIES}") - set_target_properties(glog::glog PROPERTIES - INTERFACE_COMPILE_DEFINITIONS "GLOG_USE_GLOG_EXPORT") - - find_package(Gflags) - if(GFLAGS_FOUND) - message(STATUS "Found gflags as a dependency of glog::glog, include=${LIBGFLAGS_INCLUDE_DIR}, libs=${LIBGFLAGS_LIBRARY}") - set_property(TARGET glog::glog APPEND PROPERTY IMPORTED_LINK_INTERFACE_LIBRARIES ${LIBGFLAGS_LIBRARY}) - endif() - - find_package(LibUnwind) - if(LIBUNWIND_FOUND) - message(STATUS "Found LibUnwind as a dependency of glog::glog, include=${LIBUNWIND_INCLUDE_DIR}, libs=${LIBUNWIND_LIBRARY}") - set_property(TARGET glog::glog APPEND PROPERTY IMPORTED_LINK_INTERFACE_LIBRARIES ${LIBUNWIND_LIBRARY}) - endif() -endif() diff --git a/build/fbcode_builder/CMake/FindLMDB.cmake b/build/fbcode_builder/CMake/FindLMDB.cmake deleted file mode 100644 index 51635e36e..000000000 --- a/build/fbcode_builder/CMake/FindLMDB.cmake +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This software may be used and distributed according to the terms of the -# GNU General Public License version 2. - -find_library(LMDB_LIBRARIES NAMES lmdb liblmdb) -mark_as_advanced(LMDB_LIBRARIES) - -find_path(LMDB_INCLUDE_DIR NAMES lmdb.h) -mark_as_advanced(LMDB_INCLUDE_DIR) - -find_package_handle_standard_args( - LMDB - REQUIRED_VARS LMDB_LIBRARIES LMDB_INCLUDE_DIR) - -if(LMDB_FOUND) - set(LMDB_LIBRARIES ${LMDB_LIBRARIES}) - set(LMDB_INCLUDE_DIR, ${LMDB_INCLUDE_DIR}) -endif() diff --git a/build/fbcode_builder/CMake/FindLibEvent.cmake b/build/fbcode_builder/CMake/FindLibEvent.cmake deleted file mode 100644 index dd11ebd84..000000000 --- a/build/fbcode_builder/CMake/FindLibEvent.cmake +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# - Find LibEvent (a cross event library) -# This module defines -# LIBEVENT_INCLUDE_DIR, where to find LibEvent headers -# LIBEVENT_LIB, LibEvent libraries -# LibEvent_FOUND, If false, do not try to use libevent - -set(LibEvent_EXTRA_PREFIXES /usr/local /opt/local "$ENV{HOME}") -foreach(prefix ${LibEvent_EXTRA_PREFIXES}) - list(APPEND LibEvent_INCLUDE_PATHS "${prefix}/include") - list(APPEND LibEvent_LIB_PATHS "${prefix}/lib") -endforeach() - -find_package(Libevent CONFIG QUIET) -if (TARGET event) - # Re-export the config under our own names - - # Somewhat gross, but some vcpkg installed libevents have a relative - # `include` path exported into LIBEVENT_INCLUDE_DIRS, which triggers - # a cmake error because it resolves to the `include` dir within the - # folly repo, which is not something cmake allows to be in the - # INTERFACE_INCLUDE_DIRECTORIES. Thankfully on such a system the - # actual include directory is already part of the global include - # directories, so we can just skip it. - if (NOT "${LIBEVENT_INCLUDE_DIRS}" STREQUAL "include") - set(LIBEVENT_INCLUDE_DIR ${LIBEVENT_INCLUDE_DIRS}) - else() - set(LIBEVENT_INCLUDE_DIR) - endif() - - # Unfortunately, with a bare target name `event`, downstream consumers - # of the package that depends on `Libevent` located via CONFIG end - # up exporting just a bare `event` in their libraries. This is problematic - # because this in interpreted as just `-levent` with no library path. - # When libevent is not installed in the default installation prefix - # this results in linker errors. - # To resolve this, we ask cmake to lookup the full path to the library - # and use that instead. - cmake_policy(PUSH) - if(POLICY CMP0026) - # Allow reading the LOCATION property - cmake_policy(SET CMP0026 OLD) - endif() - get_target_property(LIBEVENT_LIB event LOCATION) - cmake_policy(POP) - - set(LibEvent_FOUND ${Libevent_FOUND}) - if (NOT LibEvent_FIND_QUIETLY) - message(STATUS "Found libevent from package config include=${LIBEVENT_INCLUDE_DIRS} lib=${LIBEVENT_LIB}") - endif() -else() - find_path(LIBEVENT_INCLUDE_DIR event.h PATHS ${LibEvent_INCLUDE_PATHS}) - find_library(LIBEVENT_LIB NAMES event PATHS ${LibEvent_LIB_PATHS}) - - if (LIBEVENT_LIB AND LIBEVENT_INCLUDE_DIR) - set(LibEvent_FOUND TRUE) - set(LIBEVENT_LIB ${LIBEVENT_LIB}) - else () - set(LibEvent_FOUND FALSE) - endif () - - if (LibEvent_FOUND) - if (NOT LibEvent_FIND_QUIETLY) - message(STATUS "Found libevent: ${LIBEVENT_LIB}") - endif () - else () - if (LibEvent_FIND_REQUIRED) - message(FATAL_ERROR "Could NOT find libevent.") - endif () - message(STATUS "libevent NOT found.") - endif () - - mark_as_advanced( - LIBEVENT_LIB - LIBEVENT_INCLUDE_DIR - ) -endif() diff --git a/build/fbcode_builder/CMake/FindLibUnwind.cmake b/build/fbcode_builder/CMake/FindLibUnwind.cmake deleted file mode 100644 index 9b6a05cfe..000000000 --- a/build/fbcode_builder/CMake/FindLibUnwind.cmake +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# When using prepackaged LLVM libunwind on Ubuntu, its includes are installed in a subdirectory. -find_path(LIBUNWIND_INCLUDE_DIR NAMES libunwind.h PATH_SUFFIXES libunwind) -mark_as_advanced(LIBUNWIND_INCLUDE_DIR) - -find_library(LIBUNWIND_LIBRARY NAMES unwind) -mark_as_advanced(LIBUNWIND_LIBRARY) - -include(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS( - LIBUNWIND - REQUIRED_VARS LIBUNWIND_LIBRARY LIBUNWIND_INCLUDE_DIR) - -if(LIBUNWIND_FOUND) - set(LIBUNWIND_LIBRARIES ${LIBUNWIND_LIBRARY}) - set(LIBUNWIND_INCLUDE_DIRS ${LIBUNWIND_INCLUDE_DIR}) -endif() diff --git a/build/fbcode_builder/CMake/FindLibiberty.cmake b/build/fbcode_builder/CMake/FindLibiberty.cmake deleted file mode 100644 index d5f70ec68..000000000 --- a/build/fbcode_builder/CMake/FindLibiberty.cmake +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -find_path(LIBIBERTY_INCLUDE_DIR NAMES libiberty.h PATH_SUFFIXES libiberty) -mark_as_advanced(LIBIBERTY_INCLUDE_DIR) - -find_library(LIBIBERTY_LIBRARY NAMES iberty) -mark_as_advanced(LIBIBERTY_LIBRARY) - -include(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS( - LIBIBERTY - REQUIRED_VARS LIBIBERTY_LIBRARY LIBIBERTY_INCLUDE_DIR) - -if(LIBIBERTY_FOUND) - set(LIBIBERTY_LIBRARIES ${LIBIBERTY_LIBRARY}) - set(LIBIBERTY_INCLUDE_DIRS ${LIBIBERTY_INCLUDE_DIR}) -endif() diff --git a/build/fbcode_builder/CMake/FindPCRE.cmake b/build/fbcode_builder/CMake/FindPCRE.cmake deleted file mode 100644 index 32ccb3725..000000000 --- a/build/fbcode_builder/CMake/FindPCRE.cmake +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -include(FindPackageHandleStandardArgs) -find_path(PCRE_INCLUDE_DIR NAMES pcre.h) -find_library(PCRE_LIBRARY NAMES pcre) -find_package_handle_standard_args( - PCRE - DEFAULT_MSG - PCRE_LIBRARY - PCRE_INCLUDE_DIR -) -mark_as_advanced(PCRE_INCLUDE_DIR PCRE_LIBRARY) diff --git a/build/fbcode_builder/CMake/FindPCRE2.cmake b/build/fbcode_builder/CMake/FindPCRE2.cmake deleted file mode 100644 index c2c64a29b..000000000 --- a/build/fbcode_builder/CMake/FindPCRE2.cmake +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -include(FindPackageHandleStandardArgs) -find_path(PCRE2_INCLUDE_DIR NAMES pcre2.h) -find_library(PCRE2_LIBRARY NAMES pcre2-8) -find_package_handle_standard_args( - PCRE2 - DEFAULT_MSG - PCRE2_LIBRARY - PCRE2_INCLUDE_DIR -) -set(PCRE2_DEFINES "PCRE2_CODE_UNIT_WIDTH=8") -mark_as_advanced(PCRE2_INCLUDE_DIR PCRE2_LIBRARY PCRE2_DEFINES) diff --git a/build/fbcode_builder/CMake/FindRe2.cmake b/build/fbcode_builder/CMake/FindRe2.cmake deleted file mode 100644 index 013ae7761..000000000 --- a/build/fbcode_builder/CMake/FindRe2.cmake +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This software may be used and distributed according to the terms of the -# GNU General Public License version 2. - -find_library(RE2_LIBRARY re2) -mark_as_advanced(RE2_LIBRARY) - -find_path(RE2_INCLUDE_DIR NAMES re2/re2.h) -mark_as_advanced(RE2_INCLUDE_DIR) - -include(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS( - RE2 - REQUIRED_VARS RE2_LIBRARY RE2_INCLUDE_DIR) - -if(RE2_FOUND) - set(RE2_LIBRARY ${RE2_LIBRARY}) - set(RE2_INCLUDE_DIR, ${RE2_INCLUDE_DIR}) -endif() diff --git a/build/fbcode_builder/CMake/FindSodium.cmake b/build/fbcode_builder/CMake/FindSodium.cmake deleted file mode 100644 index 3c3f1245c..000000000 --- a/build/fbcode_builder/CMake/FindSodium.cmake +++ /dev/null @@ -1,297 +0,0 @@ -# Written in 2016 by Henrik Steffen Gaßmann -# -# To the extent possible under law, the author(s) have dedicated all -# copyright and related and neighboring rights to this software to the -# public domain worldwide. This software is distributed without any warranty. -# -# You should have received a copy of the CC0 Public Domain Dedication -# along with this software. If not, see -# -# http://creativecommons.org/publicdomain/zero/1.0/ -# -######################################################################## -# Tries to find the local libsodium installation. -# -# On Windows the sodium_DIR environment variable is used as a default -# hint which can be overridden by setting the corresponding cmake variable. -# -# Once done the following variables will be defined: -# -# sodium_FOUND -# sodium_INCLUDE_DIR -# sodium_LIBRARY_DEBUG -# sodium_LIBRARY_RELEASE -# -# -# Furthermore an imported "sodium" target is created. -# - -if (CMAKE_C_COMPILER_ID STREQUAL "GNU" - OR CMAKE_C_COMPILER_ID STREQUAL "Clang") - set(_GCC_COMPATIBLE 1) -endif() - -# static library option -if (NOT DEFINED sodium_USE_STATIC_LIBS) - option(sodium_USE_STATIC_LIBS "enable to statically link against sodium" OFF) -endif() -if(NOT (sodium_USE_STATIC_LIBS EQUAL sodium_USE_STATIC_LIBS_LAST)) - unset(sodium_LIBRARY CACHE) - unset(sodium_LIBRARY_DEBUG CACHE) - unset(sodium_LIBRARY_RELEASE CACHE) - unset(sodium_DLL_DEBUG CACHE) - unset(sodium_DLL_RELEASE CACHE) - set(sodium_USE_STATIC_LIBS_LAST ${sodium_USE_STATIC_LIBS} CACHE INTERNAL "internal change tracking variable") -endif() - - -######################################################################## -# UNIX -if (UNIX) - # import pkg-config - find_package(PkgConfig QUIET) - if (PKG_CONFIG_FOUND) - pkg_check_modules(sodium_PKG QUIET libsodium) - endif() - - if(sodium_USE_STATIC_LIBS) - foreach(_libname ${sodium_PKG_STATIC_LIBRARIES}) - if (NOT _libname MATCHES "^lib.*\\.a$") # ignore strings already ending with .a - list(INSERT sodium_PKG_STATIC_LIBRARIES 0 "lib${_libname}.a") - endif() - endforeach() - list(REMOVE_DUPLICATES sodium_PKG_STATIC_LIBRARIES) - - # if pkgconfig for libsodium doesn't provide - # static lib info, then override PKG_STATIC here.. - if (NOT sodium_PKG_STATIC_FOUND) - set(sodium_PKG_STATIC_LIBRARIES libsodium.a) - endif() - - set(XPREFIX sodium_PKG_STATIC) - else() - if (NOT sodium_PKG_FOUND) - set(sodium_PKG_LIBRARIES sodium) - endif() - - set(XPREFIX sodium_PKG) - endif() - - find_path(sodium_INCLUDE_DIR sodium.h - HINTS ${${XPREFIX}_INCLUDE_DIRS} - ) - find_library(sodium_LIBRARY_DEBUG NAMES ${${XPREFIX}_LIBRARIES} - HINTS ${${XPREFIX}_LIBRARY_DIRS} - ) - find_library(sodium_LIBRARY_RELEASE NAMES ${${XPREFIX}_LIBRARIES} - HINTS ${${XPREFIX}_LIBRARY_DIRS} - ) - - -######################################################################## -# Windows -elseif (WIN32) - set(sodium_DIR "$ENV{sodium_DIR}" CACHE FILEPATH "sodium install directory") - mark_as_advanced(sodium_DIR) - - find_path(sodium_INCLUDE_DIR sodium.h - HINTS ${sodium_DIR} - PATH_SUFFIXES include - ) - - if (MSVC) - # detect target architecture - file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/arch.cpp" [=[ - #if defined _M_IX86 - #error ARCH_VALUE x86_32 - #elif defined _M_X64 - #error ARCH_VALUE x86_64 - #endif - #error ARCH_VALUE unknown - ]=]) - try_compile(_UNUSED_VAR "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/arch.cpp" - OUTPUT_VARIABLE _COMPILATION_LOG - ) - string(REGEX REPLACE ".*ARCH_VALUE ([a-zA-Z0-9_]+).*" "\\1" _TARGET_ARCH "${_COMPILATION_LOG}") - - # construct library path - if (_TARGET_ARCH STREQUAL "x86_32") - string(APPEND _PLATFORM_PATH "Win32") - elseif(_TARGET_ARCH STREQUAL "x86_64") - string(APPEND _PLATFORM_PATH "x64") - else() - message(FATAL_ERROR "the ${_TARGET_ARCH} architecture is not supported by Findsodium.cmake.") - endif() - string(APPEND _PLATFORM_PATH "/$$CONFIG$$") - - if (MSVC_VERSION LESS 1900) - math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 60") - else() - math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 50") - endif() - string(APPEND _PLATFORM_PATH "/v${_VS_VERSION}") - - if (sodium_USE_STATIC_LIBS) - string(APPEND _PLATFORM_PATH "/static") - else() - string(APPEND _PLATFORM_PATH "/dynamic") - endif() - - string(REPLACE "$$CONFIG$$" "Debug" _DEBUG_PATH_SUFFIX "${_PLATFORM_PATH}") - string(REPLACE "$$CONFIG$$" "Release" _RELEASE_PATH_SUFFIX "${_PLATFORM_PATH}") - - find_library(sodium_LIBRARY_DEBUG libsodium.lib - HINTS ${sodium_DIR} - PATH_SUFFIXES ${_DEBUG_PATH_SUFFIX} - ) - find_library(sodium_LIBRARY_RELEASE libsodium.lib - HINTS ${sodium_DIR} - PATH_SUFFIXES ${_RELEASE_PATH_SUFFIX} - ) - if (NOT sodium_USE_STATIC_LIBS) - set(CMAKE_FIND_LIBRARY_SUFFIXES_BCK ${CMAKE_FIND_LIBRARY_SUFFIXES}) - set(CMAKE_FIND_LIBRARY_SUFFIXES ".dll") - find_library(sodium_DLL_DEBUG libsodium - HINTS ${sodium_DIR} - PATH_SUFFIXES ${_DEBUG_PATH_SUFFIX} - ) - find_library(sodium_DLL_RELEASE libsodium - HINTS ${sodium_DIR} - PATH_SUFFIXES ${_RELEASE_PATH_SUFFIX} - ) - set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_BCK}) - endif() - - elseif(_GCC_COMPATIBLE) - if (sodium_USE_STATIC_LIBS) - find_library(sodium_LIBRARY_DEBUG libsodium.a - HINTS ${sodium_DIR} - PATH_SUFFIXES lib - ) - find_library(sodium_LIBRARY_RELEASE libsodium.a - HINTS ${sodium_DIR} - PATH_SUFFIXES lib - ) - else() - find_library(sodium_LIBRARY_DEBUG libsodium.dll.a - HINTS ${sodium_DIR} - PATH_SUFFIXES lib - ) - find_library(sodium_LIBRARY_RELEASE libsodium.dll.a - HINTS ${sodium_DIR} - PATH_SUFFIXES lib - ) - - file(GLOB _DLL - LIST_DIRECTORIES false - RELATIVE "${sodium_DIR}/bin" - "${sodium_DIR}/bin/libsodium*.dll" - ) - find_library(sodium_DLL_DEBUG ${_DLL} libsodium - HINTS ${sodium_DIR} - PATH_SUFFIXES bin - ) - find_library(sodium_DLL_RELEASE ${_DLL} libsodium - HINTS ${sodium_DIR} - PATH_SUFFIXES bin - ) - endif() - else() - message(FATAL_ERROR "this platform is not supported by FindSodium.cmake") - endif() - - -######################################################################## -# unsupported -else() - message(FATAL_ERROR "this platform is not supported by FindSodium.cmake") -endif() - - -######################################################################## -# common stuff - -# extract sodium version -if (sodium_INCLUDE_DIR) - set(_VERSION_HEADER "${_INCLUDE_DIR}/sodium/version.h") - if (EXISTS _VERSION_HEADER) - file(READ "${_VERSION_HEADER}" _VERSION_HEADER_CONTENT) - string(REGEX REPLACE ".*#[ \t]*define[ \t]*SODIUM_VERSION_STRING[ \t]*\"([^\n]*)\".*" "\\1" - sodium_VERSION "${_VERSION_HEADER_CONTENT}") - set(sodium_VERSION "${sodium_VERSION}" PARENT_SCOPE) - endif() -endif() - -# communicate results -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args( - Sodium # The name must be either uppercase or match the filename case. - REQUIRED_VARS - sodium_LIBRARY_RELEASE - sodium_LIBRARY_DEBUG - sodium_INCLUDE_DIR - VERSION_VAR - sodium_VERSION -) - -if(Sodium_FOUND) - set(sodium_LIBRARIES - optimized ${sodium_LIBRARY_RELEASE} debug ${sodium_LIBRARY_DEBUG}) -endif() - -# mark file paths as advanced -mark_as_advanced(sodium_INCLUDE_DIR) -mark_as_advanced(sodium_LIBRARY_DEBUG) -mark_as_advanced(sodium_LIBRARY_RELEASE) -if (WIN32) - mark_as_advanced(sodium_DLL_DEBUG) - mark_as_advanced(sodium_DLL_RELEASE) -endif() - -# create imported target -if(sodium_USE_STATIC_LIBS) - set(_LIB_TYPE STATIC) -else() - set(_LIB_TYPE SHARED) -endif() - -if(NOT TARGET sodium) - add_library(sodium ${_LIB_TYPE} IMPORTED) -endif() - -set_target_properties(sodium PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${sodium_INCLUDE_DIR}" - IMPORTED_LINK_INTERFACE_LANGUAGES "C" -) - -if (sodium_USE_STATIC_LIBS) - set_target_properties(sodium PROPERTIES - INTERFACE_COMPILE_DEFINITIONS "SODIUM_STATIC" - IMPORTED_LOCATION "${sodium_LIBRARY_RELEASE}" - IMPORTED_LOCATION_DEBUG "${sodium_LIBRARY_DEBUG}" - ) -else() - if (UNIX) - set_target_properties(sodium PROPERTIES - IMPORTED_LOCATION "${sodium_LIBRARY_RELEASE}" - IMPORTED_LOCATION_DEBUG "${sodium_LIBRARY_DEBUG}" - ) - elseif (WIN32) - set_target_properties(sodium PROPERTIES - IMPORTED_IMPLIB "${sodium_LIBRARY_RELEASE}" - IMPORTED_IMPLIB_DEBUG "${sodium_LIBRARY_DEBUG}" - ) - if (NOT (sodium_DLL_DEBUG MATCHES ".*-NOTFOUND")) - set_target_properties(sodium PROPERTIES - IMPORTED_LOCATION_DEBUG "${sodium_DLL_DEBUG}" - ) - endif() - if (NOT (sodium_DLL_RELEASE MATCHES ".*-NOTFOUND")) - set_target_properties(sodium PROPERTIES - IMPORTED_LOCATION_RELWITHDEBINFO "${sodium_DLL_RELEASE}" - IMPORTED_LOCATION_MINSIZEREL "${sodium_DLL_RELEASE}" - IMPORTED_LOCATION_RELEASE "${sodium_DLL_RELEASE}" - ) - endif() - endif() -endif() diff --git a/build/fbcode_builder/CMake/FindXxhash.cmake b/build/fbcode_builder/CMake/FindXxhash.cmake deleted file mode 100644 index 04760cbf7..000000000 --- a/build/fbcode_builder/CMake/FindXxhash.cmake +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# -# - Try to find Facebook xxhash library -# This will define -# Xxhash_FOUND -# Xxhash_INCLUDE_DIR -# Xxhash_LIBRARY -# - -find_path(Xxhash_INCLUDE_DIR NAMES xxhash.h) - -find_library(Xxhash_LIBRARY_RELEASE NAMES xxhash) - -include(SelectLibraryConfigurations) -SELECT_LIBRARY_CONFIGURATIONS(Xxhash) - -include(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS( - Xxhash DEFAULT_MSG - Xxhash_LIBRARY Xxhash_INCLUDE_DIR -) - -if (Xxhash_FOUND) - message(STATUS "Found xxhash: ${Xxhash_LIBRARY}") -endif() - -mark_as_advanced(Xxhash_INCLUDE_DIR Xxhash_LIBRARY) diff --git a/build/fbcode_builder/CMake/FindZstd.cmake b/build/fbcode_builder/CMake/FindZstd.cmake deleted file mode 100644 index 51fd6c4db..000000000 --- a/build/fbcode_builder/CMake/FindZstd.cmake +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# -# - Try to find Facebook zstd library -# This will define -# ZSTD_FOUND -# ZSTD_INCLUDE_DIR -# ZSTD_LIBRARY -# - -find_path(ZSTD_INCLUDE_DIR NAMES zstd.h) - -find_library(ZSTD_LIBRARY_DEBUG NAMES zstdd zstd_staticd) -find_library(ZSTD_LIBRARY_RELEASE NAMES zstd zstd_static) - -include(SelectLibraryConfigurations) -SELECT_LIBRARY_CONFIGURATIONS(ZSTD) - -include(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS( - Zstd DEFAULT_MSG - ZSTD_LIBRARY ZSTD_INCLUDE_DIR -) - -if (ZSTD_FOUND) - message(STATUS "Found Zstd: ${ZSTD_LIBRARY}") -endif() - -mark_as_advanced(ZSTD_INCLUDE_DIR ZSTD_LIBRARY) diff --git a/build/fbcode_builder/CMake/Findibverbs.cmake b/build/fbcode_builder/CMake/Findibverbs.cmake deleted file mode 100644 index 1a6dbc5dd..000000000 --- a/build/fbcode_builder/CMake/Findibverbs.cmake +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Find the ibverbs libraries -# -# The following variables are optionally searched for defaults -# IBVERBS_ROOT_DIR: Base directory where all ibverbs components are found -# IBVERBS_INCLUDE_DIR: Directory where ibverbs headers are found -# IBVERBS_LIB_DIR: Directory where ibverbs libraries are found - -# The following are set after configuration is done: -# IBVERBS_FOUND -# IBVERBS_INCLUDE_DIRS -# IBVERBS_LIBRARIES -# IBVERBS_VERSION - -find_path(IBVERBS_INCLUDE_DIRS - NAMES infiniband/verbs.h - HINTS - ${IBVERBS_INCLUDE_DIR} - ${IBVERBS_ROOT_DIR} - ${IBVERBS_ROOT_DIR}/include) - -find_library(IBVERBS_LIBRARIES - NAMES ibverbs - HINTS - ${IBVERBS_LIB_DIR} - ${IBVERBS_ROOT_DIR} - ${IBVERBS_ROOT_DIR}/lib) - -# Try to determine the rdma-core version -if(IBVERBS_INCLUDE_DIRS AND IBVERBS_LIBRARIES) - # First try using pkg-config if available - find_package(PkgConfig QUIET) - if(PKG_CONFIG_FOUND) - pkg_check_modules(PC_RDMA_CORE QUIET rdma-core) - if(PC_RDMA_CORE_VERSION) - set(IBVERBS_VERSION ${PC_RDMA_CORE_VERSION}) - endif() - endif() - - # If pkg-config didn't work, try to extract version from library filename - # According to rdma-core Documentation/versioning.md: - # Library filename format: - # libibverbs.so.SONAME.ABI.PACKAGE_VERSION_MAIN[.PACKAGE_VERSION_BRANCH] - # Where: - # - SONAME: Major version (1st field) - # - ABI: ABI version number (2nd field) - # - PACKAGE_VERSION_MAIN: Main package version (3rd field) - # - PACKAGE_VERSION_BRANCH: Optional counter for branched stable - # releases (4th field, part of PACKAGE_VERSION) - # Example: libibverbs.so.1.14.57.0 → SONAME=1, ABI=14, - # PACKAGE_VERSION=57.0 - if(NOT IBVERBS_VERSION) - # Get the real path of the library (follows symlinks) - get_filename_component(IBVERBS_REAL_PATH "${IBVERBS_LIBRARIES}" REALPATH) - get_filename_component(IBVERBS_LIB_NAME "${IBVERBS_REAL_PATH}" NAME) - - # Extract version from filename - if(IBVERBS_LIB_NAME MATCHES - "libibverbs\\.so\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)") - # Four-component version: PACKAGE_VERSION_MAIN.PACKAGE_VERSION_BRANCH - set(IBVERBS_VERSION_MAJOR ${CMAKE_MATCH_3}) - set(IBVERBS_VERSION_MINOR ${CMAKE_MATCH_4}) - set(IBVERBS_VERSION "${IBVERBS_VERSION_MAJOR}.${IBVERBS_VERSION_MINOR}") - elseif(IBVERBS_LIB_NAME MATCHES - "libibverbs\\.so\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)") - # Three-component version: PACKAGE_VERSION_MAIN only - set(IBVERBS_VERSION_MAJOR ${CMAKE_MATCH_3}) - set(IBVERBS_VERSION "${IBVERBS_VERSION_MAJOR}.0") - else() - # If we can't parse the filename, set to empty string - # Feature detection will be done in CMakeLists.txt - set(IBVERBS_VERSION "") - endif() - endif() -endif() - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(ibverbs - REQUIRED_VARS IBVERBS_INCLUDE_DIRS IBVERBS_LIBRARIES - VERSION_VAR IBVERBS_VERSION) -mark_as_advanced(IBVERBS_INCLUDE_DIRS IBVERBS_LIBRARIES IBVERBS_VERSION) diff --git a/build/fbcode_builder/CMake/RustStaticLibrary.cmake b/build/fbcode_builder/CMake/RustStaticLibrary.cmake deleted file mode 100644 index 04f946684..000000000 --- a/build/fbcode_builder/CMake/RustStaticLibrary.cmake +++ /dev/null @@ -1,537 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. - -include(FBCMakeParseArgs) - -set( - USE_CARGO_VENDOR AUTO CACHE STRING - "Download Rust Crates from an internally vendored location" -) -set_property(CACHE USE_CARGO_VENDOR PROPERTY STRINGS AUTO ON OFF) - -set( - GENERATE_CARGO_VENDOR_CONFIG AUTO CACHE STRING - "Whether to generate Rust cargo vendor config or use existing" -) -set_property(CACHE GENERATE_CARGO_VENDOR_CONFIG PROPERTY STRINGS AUTO ON OFF) - -set(RUST_VENDORED_CRATES_DIR "$ENV{RUST_VENDORED_CRATES_DIR}") - -if("${USE_CARGO_VENDOR}" STREQUAL "AUTO") - if(EXISTS "${RUST_VENDORED_CRATES_DIR}") - set(USE_CARGO_VENDOR ON) - else() - set(USE_CARGO_VENDOR OFF) - endif() -endif() - -if("${GENERATE_CARGO_VENDOR_CONFIG}" STREQUAL "AUTO") - set(GENERATE_CARGO_VENDOR_CONFIG "${USE_CARGO_VENDOR}") -endif() - -if(GENERATE_CARGO_VENDOR_CONFIG) - if(NOT EXISTS "${RUST_VENDORED_CRATES_DIR}") - message( - FATAL "vendored rust crates not present: " - "${RUST_VENDORED_CRATES_DIR}" - ) - endif() - - set(RUST_CARGO_HOME "${CMAKE_BINARY_DIR}/_cargo_home") - file(MAKE_DIRECTORY "${RUST_CARGO_HOME}") - - file( - TO_NATIVE_PATH "${RUST_VENDORED_CRATES_DIR}" - ESCAPED_RUST_VENDORED_CRATES_DIR - ) - string( - REPLACE "\\" "\\\\" - ESCAPED_RUST_VENDORED_CRATES_DIR - "${ESCAPED_RUST_VENDORED_CRATES_DIR}" - ) - file( - WRITE "${RUST_CARGO_HOME}/config" - "[source.crates-io]\n" - "replace-with = \"vendored-sources\"\n" - "\n" - "[source.vendored-sources]\n" - "directory = \"${ESCAPED_RUST_VENDORED_CRATES_DIR}\"\n" - ) -endif() - -find_program(CARGO_COMMAND cargo REQUIRED) - -# Cargo is a build system in itself, and thus will try to take advantage of all -# the cores on the system. Unfortunately, this conflicts with Ninja, since it -# also tries to utilize all the cores. This can lead to a system that is -# completely overloaded with compile jobs to the point where nothing else can -# be achieved on the system. -# -# Let's inform Ninja of this fact so it won't try to spawn other jobs while -# Rust being compiled. -set_property(GLOBAL APPEND PROPERTY JOB_POOLS rust_job_pool=1) - -# This function creates an interface library target based on the static library -# built by Cargo. It will call Cargo to build a staticlib and generate a CMake -# interface library with it. -# -# This function requires `find_package(Python COMPONENTS Interpreter)`. -# -# You need to set `lib:crate-type = ["staticlib"]` in your Cargo.toml to make -# Cargo build static library. -# -# ```cmake -# rust_static_library( [CRATE ] [FEATURES ] [USE_CXX_INCLUDE]) -# ``` -# -# Parameters: -# - TARGET: -# Name of the target name. This function will create an interface library -# target with this name. -# - CRATE_NAME: -# Name of the crate. This parameter is optional. If unspecified, it will -# fallback to `${TARGET}`. -# - FEATURE_NAME: -# Name of the Rust feature to enable. -# - USE_CXX_INCLUDE: -# Include cxx.rs include path in `${TARGET}` INTERFACE. -# -# This function creates two targets: -# - "${TARGET}": an interface library target contains the static library built -# from Cargo. -# - "${TARGET}.cargo": an internal custom target that invokes Cargo. -# -# If you are going to use this static library from C/C++, you will need to -# write header files for the library (or generate with cbindgen) and bind these -# headers with the interface library. -# -function(rust_static_library TARGET) - fb_cmake_parse_args(ARG "USE_CXX_INCLUDE" "CRATE;FEATURES" "" "${ARGN}") - - if(DEFINED ARG_CRATE) - set(crate_name "${ARG_CRATE}") - else() - set(crate_name "${TARGET}") - endif() - if(DEFINED ARG_FEATURES) - set(features --features ${ARG_FEATURES}) - else() - set(features ) - endif() - - set(cargo_target "${TARGET}.cargo") - set(target_dir $,debug,release>) - set(staticlib_name "${CMAKE_STATIC_LIBRARY_PREFIX}${crate_name}${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(rust_staticlib "${CMAKE_CURRENT_BINARY_DIR}/${target_dir}/${staticlib_name}") - - if(DEFINED ARG_FEATURES) - set(cargo_flags build $,,--release> -p ${crate_name} --features ${ARG_FEATURES} --config fbcode_build=false) - else() - set(cargo_flags build $,,--release> -p ${crate_name} --config fbcode_build=false) - endif() - if(USE_CARGO_VENDOR) - set(extra_cargo_env "CARGO_HOME=${RUST_CARGO_HOME}") - set(cargo_flags ${cargo_flags}) - endif() - - add_custom_target( - ${cargo_target} - COMMAND - "${CMAKE_COMMAND}" -E remove -f "${CMAKE_CURRENT_SOURCE_DIR}/Cargo.lock" - COMMAND - "${CMAKE_COMMAND}" -E env - "CARGO_TARGET_DIR=${CMAKE_CURRENT_BINARY_DIR}" - ${extra_cargo_env} - ${CARGO_COMMAND} - ${cargo_flags} - COMMENT "Building Rust crate '${crate_name}'..." - JOB_POOL rust_job_pool - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - BYPRODUCTS - "${CMAKE_CURRENT_BINARY_DIR}/debug/${staticlib_name}" - "${CMAKE_CURRENT_BINARY_DIR}/release/${staticlib_name}" - ) - - add_library(${TARGET} INTERFACE) - add_dependencies(${TARGET} ${cargo_target}) - set_target_properties( - ${TARGET} - PROPERTIES - INTERFACE_STATICLIB_OUTPUT_PATH "${rust_staticlib}" - INTERFACE_INSTALL_LIBNAME - "${CMAKE_STATIC_LIBRARY_PREFIX}${crate_name}_rs${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - - if(DEFINED ARG_USE_CXX_INCLUDE) - target_include_directories( - ${TARGET} - INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/cxxbridge/ - ) - endif() - - target_link_libraries( - ${TARGET} - INTERFACE "$" - ) -endfunction() - -# This function instructs CMake to define a target that will use `cargo build` -# to build a bin crate referenced by the Cargo.toml file in the current source -# directory. -# It accepts a single `TARGET` parameter which will be passed as the package -# name to `cargo build -p TARGET`. If binary has different name as package, -# use optional flag BINARY_NAME to override it. -# It also accepts a `FEATURES` parameter if you want to enable certain features -# in your Rust binary. -# The CMake target will be registered to build by default as part of the -# ALL target. -function(rust_executable TARGET) - fb_cmake_parse_args(ARG "" "BINARY_NAME;FEATURES" "" "${ARGN}") - - set(crate_name "${TARGET}") - set(cargo_target "${TARGET}.cargo") - set(target_dir $,debug,release>) - - if(DEFINED ARG_BINARY_NAME) - set(executable_name "${ARG_BINARY_NAME}${CMAKE_EXECUTABLE_SUFFIX}") - else() - set(executable_name "${crate_name}${CMAKE_EXECUTABLE_SUFFIX}") - endif() - if(DEFINED ARG_FEATURES) - set(features --features ${ARG_FEATURES}) - else() - set(features ) - endif() - - if(DEFINED ARG_FEATURES) - set(cargo_flags build $,,--release> -p ${crate_name} --features ${ARG_FEATURES}) - else() - set(cargo_flags build $,,--release> -p ${crate_name}) - endif() - if(USE_CARGO_VENDOR) - set(extra_cargo_env "CARGO_HOME=${RUST_CARGO_HOME}") - set(cargo_flags ${cargo_flags}) - endif() - - add_custom_target( - ${cargo_target} - ALL - COMMAND - "${CMAKE_COMMAND}" -E remove -f "${CMAKE_CURRENT_SOURCE_DIR}/Cargo.lock" - COMMAND - "${CMAKE_COMMAND}" -E env - "CARGO_TARGET_DIR=${CMAKE_CURRENT_BINARY_DIR}" - ${extra_cargo_env} - ${CARGO_COMMAND} - ${cargo_flags} - COMMENT "Building Rust executable '${crate_name}'..." - JOB_POOL rust_job_pool - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - BYPRODUCTS - "${CMAKE_CURRENT_BINARY_DIR}/debug/${executable_name}" - "${CMAKE_CURRENT_BINARY_DIR}/release/${executable_name}" - ) - - set_property(TARGET "${cargo_target}" - PROPERTY EXECUTABLE "${CMAKE_CURRENT_BINARY_DIR}/${target_dir}/${executable_name}") -endfunction() - -# This function can be used to install the executable generated by a prior -# call to the `rust_executable` function. -# It requires a `TARGET` parameter to identify the target to be installed, -# and an optional `DESTINATION` parameter to specify the installation -# directory. If DESTINATION is not specified then the `bin` directory -# will be assumed. -function(install_rust_executable TARGET) - # Parse the arguments - set(one_value_args DESTINATION) - set(multi_value_args) - fb_cmake_parse_args( - ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}" - ) - - if(NOT DEFINED ARG_DESTINATION) - set(ARG_DESTINATION bin) - endif() - - get_target_property(foo "${TARGET}.cargo" EXECUTABLE) - - install( - PROGRAMS "${foo}" - DESTINATION "${ARG_DESTINATION}" - ) -endfunction() - -# This function installs the interface target generated from the function -# `rust_static_library`. Use this function if you want to export your Rust -# target to external CMake targets. -# -# ```cmake -# install_rust_static_library( -# -# INSTALL_DIR -# [EXPORT ] -# ) -# ``` -# -# Parameters: -# - TARGET: Name of the Rust static library target. -# - EXPORT_NAME: Name of the exported target. -# - INSTALL_DIR: Path to the directory where this library will be installed. -# -function(install_rust_static_library TARGET) - fb_cmake_parse_args(ARG "" "EXPORT;INSTALL_DIR" "" "${ARGN}") - - get_property( - staticlib_output_path - TARGET "${TARGET}" - PROPERTY INTERFACE_STATICLIB_OUTPUT_PATH - ) - get_property( - staticlib_output_name - TARGET "${TARGET}" - PROPERTY INTERFACE_INSTALL_LIBNAME - ) - - if(NOT DEFINED staticlib_output_path) - message(FATAL_ERROR "Not a rust_static_library target.") - endif() - - if(NOT DEFINED ARG_INSTALL_DIR) - message(FATAL_ERROR "Missing required argument.") - endif() - - if(DEFINED ARG_EXPORT) - set(install_export_args EXPORT "${ARG_EXPORT}") - endif() - - set(install_interface_dir "${ARG_INSTALL_DIR}") - if(NOT IS_ABSOLUTE "${install_interface_dir}") - set(install_interface_dir "\${_IMPORT_PREFIX}/${install_interface_dir}") - endif() - - target_link_libraries( - ${TARGET} INTERFACE - "$" - ) - install( - TARGETS ${TARGET} - ${install_export_args} - LIBRARY DESTINATION ${ARG_INSTALL_DIR} - ) - install( - FILES ${staticlib_output_path} - RENAME ${staticlib_output_name} - DESTINATION ${ARG_INSTALL_DIR} - ) -endfunction() - -# This function creates C++ bindings using the [cxx] crate. -# -# Original function found here: https://github.com/corrosion-rs/corrosion/blob/master/cmake/Corrosion.cmake#L1390 -# Simplified for use as part of RustStaticLibrary module. License below. -# -# MIT License -# -# Copyright (c) 2018 Andrew Gaspar -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# The rules approximately do the following: -# - Check which version of `cxx` the Rust crate depends on. -# - Check if the exact same version of `cxxbridge-cmd` is installed -# - If not, create a rule to build the exact same version of `cxxbridge-cmd`. -# - Create rules to run `cxxbridge` and generate -# - The `rust/cxx.h` header -# - A header and source file for the specified CXX_BRIDGE_FILE. -# - The generated sources (and header include directories) are added to the -# `${TARGET}` CMake library target. -# -# ```cmake -# rust_cxx_bridge( [CRATE ] [LIBS ]) -# ``` -# -# Parameters: -# - TARGET: -# Name of the target name. The target that the bridge will be included with. -# - CXX_BRIDGE_FILE: -# Name of the file that include the cxxbridge (e.g., "src/ffi.rs"). -# - CRATE_NAME: -# Name of the crate. This parameter is optional. If unspecified, it will -# fallback to `${TARGET}`. -# - LIBS [ ...]: -# A list of libraries that this library depends on. -# -function(rust_cxx_bridge TARGET CXX_BRIDGE_FILE) - fb_cmake_parse_args(ARG "" "CRATE" "LIBS" "${ARGN}") - - if(DEFINED ARG_CRATE) - set(crate_name "${ARG_CRATE}") - else() - set(crate_name "${TARGET}") - endif() - - if(USE_CARGO_VENDOR) - set(extra_cargo_env "CARGO_HOME=${RUST_CARGO_HOME}") - endif() - - execute_process( - COMMAND - "${CMAKE_COMMAND}" -E env - ${extra_cargo_env} - "${CARGO_COMMAND}" tree -i cxx --depth=0 - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - RESULT_VARIABLE cxx_version_result - OUTPUT_VARIABLE cxx_version_output - ) - - if(NOT "${cxx_version_result}" EQUAL "0") - message(FATAL_ERROR "Crate ${crate_name} does not depend on cxx.") - endif() - if(cxx_version_output MATCHES "cxx v([0-9]+.[0-9]+.[0-9]+)") - set(cxx_required_version "${CMAKE_MATCH_1}") - else() - message( - FATAL_ERROR - "Failed to parse cxx version from cargo tree output: `cxx_version_output`") - endif() - - # First check if a suitable version of cxxbridge is installed - find_program(INSTALLED_CXXBRIDGE cxxbridge PATHS "$ENV{HOME}/.cargo/bin/") - mark_as_advanced(INSTALLED_CXXBRIDGE) - if(INSTALLED_CXXBRIDGE) - execute_process( - COMMAND "${INSTALLED_CXXBRIDGE}" --version - OUTPUT_VARIABLE cxxbridge_version_output - ) - if(cxxbridge_version_output MATCHES "cxxbridge ([0-9]+.[0-9]+.[0-9]+)") - set(cxxbridge_version "${CMAKE_MATCH_1}") - else() - set(cxxbridge_version "") - endif() - endif() - - set(cxxbridge "") - if(cxxbridge_version) - if(cxxbridge_version VERSION_EQUAL cxx_required_version) - set(cxxbridge "${INSTALLED_CXXBRIDGE}") - if(NOT TARGET "cxxbridge_v${cxx_required_version}") - # Add an empty target. - add_custom_target("cxxbridge_v${cxx_required_version}") - endif() - endif() - endif() - - # No suitable version of cxxbridge was installed, - # so use custom target to install correct version. - if(NOT cxxbridge) - if(NOT TARGET "cxxbridge_v${cxx_required_version}") - add_custom_command( - OUTPUT - "${CMAKE_BINARY_DIR}/cxxbridge_v${cxx_required_version}/bin/cxxbridge" - COMMAND - "${CMAKE_COMMAND}" -E make_directory - "${CMAKE_BINARY_DIR}/cxxbridge_v${cxx_required_version}" - COMMAND - "${CMAKE_COMMAND}" -E remove -f "${CMAKE_CURRENT_SOURCE_DIR}/Cargo.lock" - COMMAND - "${CMAKE_COMMAND}" -E env - ${extra_cargo_env} - "${CARGO_COMMAND}" install cxxbridge-cmd - --version "${cxx_required_version}" - --root "${CMAKE_BINARY_DIR}/cxxbridge_v${cxx_required_version}" - --quiet - COMMAND - "${CMAKE_COMMAND}" -E remove -f "${CMAKE_CURRENT_SOURCE_DIR}/Cargo.lock" - COMMENT "Installing cxxbridge (version ${cxx_required_version})" - ) - add_custom_target( - "cxxbridge_v${cxx_required_version}" - DEPENDS "${CMAKE_BINARY_DIR}/cxxbridge_v${cxx_required_version}/bin/cxxbridge" - ) - endif() - set( - cxxbridge - "${CMAKE_BINARY_DIR}/cxxbridge_v${cxx_required_version}/bin/cxxbridge" - ) - endif() - - add_library(${crate_name} STATIC) - target_include_directories( - ${crate_name} - PUBLIC - $ - $ - ) - target_link_libraries( - ${crate_name} - PUBLIC - ${ARG_LIBS} - ) - - file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/rust") - add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/rust/cxx.h" - COMMAND - "${cxxbridge}" --header --output "${CMAKE_CURRENT_BINARY_DIR}/rust/cxx.h" - DEPENDS "cxxbridge_v${cxx_required_version}" - COMMENT "Generating rust/cxx.h header" - ) - - get_filename_component(filename_component ${CXX_BRIDGE_FILE} NAME) - get_filename_component(directory_component ${CXX_BRIDGE_FILE} DIRECTORY) - set(directory "") - if(directory_component) - set(directory "${directory_component}") - endif() - - set(cxx_header ${directory}/${filename_component}.h) - set(cxx_source ${directory}/${filename_component}.cc) - set(rust_source_path "${CMAKE_CURRENT_SOURCE_DIR}/${CXX_BRIDGE_FILE}") - - file( - MAKE_DIRECTORY - "${CMAKE_CURRENT_BINARY_DIR}/${directory_component}" - ) - - add_custom_command( - OUTPUT - "${CMAKE_CURRENT_BINARY_DIR}/${cxx_header}" - "${CMAKE_CURRENT_BINARY_DIR}/${cxx_source}" - COMMAND - ${cxxbridge} ${rust_source_path} - --cfg fbcode_build=false - --header - --output "${CMAKE_CURRENT_BINARY_DIR}/${cxx_header}" - COMMAND - ${cxxbridge} ${rust_source_path} - --cfg fbcode_build=false - --output "${CMAKE_CURRENT_BINARY_DIR}/${cxx_source}" - --include "${cxx_header}" - DEPENDS "cxxbridge_v${cxx_required_version}" "${rust_source_path}" - COMMENT "Generating cxx bindings for crate ${crate_name}" - ) - - target_sources( - ${crate_name} - PRIVATE - "${CMAKE_CURRENT_BINARY_DIR}/${cxx_header}" - "${CMAKE_CURRENT_BINARY_DIR}/rust/cxx.h" - "${CMAKE_CURRENT_BINARY_DIR}/${cxx_source}" - ) -endfunction() diff --git a/build/fbcode_builder/CMake/fb_py_test_main.py b/build/fbcode_builder/CMake/fb_py_test_main.py deleted file mode 100644 index a9499e221..000000000 --- a/build/fbcode_builder/CMake/fb_py_test_main.py +++ /dev/null @@ -1,805 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) Facebook, Inc. and its affiliates. -# -""" -This file contains the main module code for Python test programs. -""" - - -import contextlib -import ctypes -import fnmatch -import json -import logging -import optparse -import os -import platform -import re -import sys -import tempfile -import time -import traceback -import unittest -import warnings -from importlib.machinery import PathFinder - - -try: - from StringIO import StringIO -except ImportError: - from io import StringIO -try: - import coverage -except ImportError: - coverage = None # type: ignore -try: - from importlib.machinery import SourceFileLoader -except ImportError: - SourceFileLoader = None # type: ignore - - -class get_cpu_instr_counter(object): - def read(self): - # TODO - return 0 - - -EXIT_CODE_SUCCESS = 0 -EXIT_CODE_TEST_FAILURE = 70 - - -class TestStatus(object): - - ABORTED = "FAILURE" - PASSED = "SUCCESS" - FAILED = "FAILURE" - EXPECTED_FAILURE = "SUCCESS" - UNEXPECTED_SUCCESS = "FAILURE" - SKIPPED = "ASSUMPTION_VIOLATION" - - -class PathMatcher(object): - def __init__(self, include_patterns, omit_patterns): - self.include_patterns = include_patterns - self.omit_patterns = omit_patterns - - def omit(self, path): - """ - Omit iff matches any of the omit_patterns or the include patterns are - not empty and none is matched - """ - path = os.path.realpath(path) - return any(fnmatch.fnmatch(path, p) for p in self.omit_patterns) or ( - self.include_patterns - and not any(fnmatch.fnmatch(path, p) for p in self.include_patterns) - ) - - def include(self, path): - return not self.omit(path) - - -class DebugWipeFinder(PathFinder): - """ - PEP 302 finder that uses a DebugWipeLoader for all files which do not need - coverage - """ - - def __init__(self, matcher): - self.matcher = matcher - - def find_spec(self, fullname, path=None, target=None): - spec = super().find_spec(fullname, path=path, target=target) - if spec is None or spec.origin is None: - return None - if not spec.origin.endswith(".py"): - return None - if self.matcher.include(spec.origin): - return None - - class PyVarObject(ctypes.Structure): - _fields_ = [ - ("ob_refcnt", ctypes.c_long), - ("ob_type", ctypes.c_void_p), - ("ob_size", ctypes.c_ulong), - ] - - class DebugWipeLoader(SourceFileLoader): - """ - PEP302 loader that zeros out debug information before execution - """ - - def get_code(self, fullname): - code = super().get_code(fullname) - if code: - # Ideally we'd do - # code.co_lnotab = b'' - # But code objects are READONLY. Not to worry though; we'll - # directly modify CPython's object - code_impl = PyVarObject.from_address(id(code.co_lnotab)) - code_impl.ob_size = 0 - return code - - if isinstance(spec.loader, SourceFileLoader): - spec.loader = DebugWipeLoader(fullname, spec.origin) - return spec - - -def optimize_for_coverage(cov, include_patterns, omit_patterns): - """ - We get better performance if we zero out debug information for files which - we're not interested in. Only available in CPython 3.3+ - """ - matcher = PathMatcher(include_patterns, omit_patterns) - if SourceFileLoader and platform.python_implementation() == "CPython": - sys.meta_path.insert(0, DebugWipeFinder(matcher)) - - -class TeeStream(object): - def __init__(self, *streams): - self._streams = streams - - def write(self, data): - for stream in self._streams: - stream.write(data) - - def flush(self): - for stream in self._streams: - stream.flush() - - def isatty(self): - return False - - -class CallbackStream(object): - def __init__(self, callback, bytes_callback=None, orig=None): - self._callback = callback - self._fileno = orig.fileno() if orig else None - - # Python 3 APIs: - # - `encoding` is a string holding the encoding name - # - `errors` is a string holding the error-handling mode for encoding - # - `buffer` should look like an io.BufferedIOBase object - - self.errors = orig.errors if orig else None - if bytes_callback: - # those members are only on the io.TextIOWrapper - self.encoding = orig.encoding if orig else "UTF-8" - self.buffer = CallbackStream(bytes_callback, orig=orig) - - def write(self, data): - self._callback(data) - - def flush(self): - pass - - def isatty(self): - return False - - def fileno(self): - return self._fileno - - -class BuckTestResult(unittest.TextTestResult): - """ - Our own TestResult class that outputs data in a format that can be easily - parsed by buck's test runner. - """ - - _instr_counter = get_cpu_instr_counter() - - def __init__( - self, stream, descriptions, verbosity, show_output, main_program, suite - ): - super(BuckTestResult, self).__init__(stream, descriptions, verbosity) - self._main_program = main_program - self._suite = suite - self._results = [] - self._current_test = None - self._saved_stdout = sys.stdout - self._saved_stderr = sys.stderr - self._show_output = show_output - - def getResults(self): - return self._results - - def startTest(self, test): - super(BuckTestResult, self).startTest(test) - - # Pass in the real stdout and stderr filenos. We can't really do much - # here to intercept callers who directly operate on these fileno - # objects. - sys.stdout = CallbackStream( - self.addStdout, self.addStdoutBytes, orig=sys.stdout - ) - sys.stderr = CallbackStream( - self.addStderr, self.addStderrBytes, orig=sys.stderr - ) - self._current_test = test - self._test_start_time = time.time() - self._current_status = TestStatus.ABORTED - self._messages = [] - self._stacktrace = None - self._stdout = "" - self._stderr = "" - self._start_instr_count = self._instr_counter.read() - - def _find_next_test(self, suite): - """ - Find the next test that has not been run. - """ - - for test in suite: - - # We identify test suites by test that are iterable (as is done in - # the builtin python test harness). If we see one, recurse on it. - if hasattr(test, "__iter__"): - test = self._find_next_test(test) - - # The builtin python test harness sets test references to `None` - # after they have run, so we know we've found the next test up - # if it's not `None`. - if test is not None: - return test - - def stopTest(self, test): - sys.stdout = self._saved_stdout - sys.stderr = self._saved_stderr - - super(BuckTestResult, self).stopTest(test) - - # If a failure occurred during module/class setup, then this "test" may - # actually be a `_ErrorHolder`, which doesn't contain explicit info - # about the upcoming test. Since we really only care about the test - # name field (i.e. `_testMethodName`), we use that to detect an actual - # test cases, and fall back to looking the test up from the suite - # otherwise. - if not hasattr(test, "_testMethodName"): - test = self._find_next_test(self._suite) - - result = { - "testCaseName": "{0}.{1}".format( - test.__class__.__module__, test.__class__.__name__ - ), - "testCase": test._testMethodName, - "type": self._current_status, - "time": int((time.time() - self._test_start_time) * 1000), - "message": os.linesep.join(self._messages), - "stacktrace": self._stacktrace, - "stdOut": self._stdout, - "stdErr": self._stderr, - } - - # TestPilot supports an instruction count field. - if "TEST_PILOT" in os.environ: - result["instrCount"] = ( - int(self._instr_counter.read() - self._start_instr_count), - ) - - self._results.append(result) - self._current_test = None - - def stopTestRun(self): - cov = self._main_program.get_coverage() - if cov is not None: - self._results.append({"coverage": cov}) - - @contextlib.contextmanager - def _withTest(self, test): - self.startTest(test) - yield - self.stopTest(test) - - def _setStatus(self, test, status, message=None, stacktrace=None): - assert test == self._current_test - self._current_status = status - self._stacktrace = stacktrace - if message is not None: - if message.endswith(os.linesep): - message = message[:-1] - self._messages.append(message) - - def setStatus(self, test, status, message=None, stacktrace=None): - # addError() may be called outside of a test if one of the shared - # fixtures (setUpClass/tearDownClass/setUpModule/tearDownModule) - # throws an error. - # - # In this case, create a fake test result to record the error. - if self._current_test is None: - with self._withTest(test): - self._setStatus(test, status, message, stacktrace) - else: - self._setStatus(test, status, message, stacktrace) - - def setException(self, test, status, excinfo): - exctype, value, tb = excinfo - self.setStatus( - test, - status, - "{0}: {1}".format(exctype.__name__, value), - "".join(traceback.format_tb(tb)), - ) - - def addSuccess(self, test): - super(BuckTestResult, self).addSuccess(test) - self.setStatus(test, TestStatus.PASSED) - - def addError(self, test, err): - super(BuckTestResult, self).addError(test, err) - self.setException(test, TestStatus.ABORTED, err) - - def addFailure(self, test, err): - super(BuckTestResult, self).addFailure(test, err) - self.setException(test, TestStatus.FAILED, err) - - def addSkip(self, test, reason): - super(BuckTestResult, self).addSkip(test, reason) - self.setStatus(test, TestStatus.SKIPPED, "Skipped: %s" % (reason,)) - - def addExpectedFailure(self, test, err): - super(BuckTestResult, self).addExpectedFailure(test, err) - self.setException(test, TestStatus.EXPECTED_FAILURE, err) - - def addUnexpectedSuccess(self, test): - super(BuckTestResult, self).addUnexpectedSuccess(test) - self.setStatus(test, TestStatus.UNEXPECTED_SUCCESS, "Unexpected success") - - def addStdout(self, val): - self._stdout += val - if self._show_output: - self._saved_stdout.write(val) - self._saved_stdout.flush() - - def addStdoutBytes(self, val): - string = val.decode("utf-8", errors="backslashreplace") - self.addStdout(string) - - def addStderr(self, val): - self._stderr += val - if self._show_output: - self._saved_stderr.write(val) - self._saved_stderr.flush() - - def addStderrBytes(self, val): - string = val.decode("utf-8", errors="backslashreplace") - self.addStderr(string) - - -class BuckTestRunner(unittest.TextTestRunner): - def __init__(self, main_program, suite, show_output=True, **kwargs): - super(BuckTestRunner, self).__init__(**kwargs) - self.show_output = show_output - self._main_program = main_program - self._suite = suite - - def _makeResult(self): - return BuckTestResult( - self.stream, - self.descriptions, - self.verbosity, - self.show_output, - self._main_program, - self._suite, - ) - - -def _format_test_name(test_class, attrname): - return "{0}.{1}.{2}".format(test_class.__module__, test_class.__name__, attrname) - - -class StderrLogHandler(logging.StreamHandler): - """ - This class is very similar to logging.StreamHandler, except that it - always uses the current sys.stderr object. - - StreamHandler caches the current sys.stderr object when it is constructed. - This makes it behave poorly in unit tests, which may replace sys.stderr - with a StringIO buffer during tests. The StreamHandler will continue using - the old sys.stderr object instead of the desired StringIO buffer. - """ - - def __init__(self): - logging.Handler.__init__(self) - - @property - def stream(self): - return sys.stderr - - -class RegexTestLoader(unittest.TestLoader): - def __init__(self, regex=None): - self.regex = regex - super(RegexTestLoader, self).__init__() - - def getTestCaseNames(self, testCaseClass): - """ - Return a sorted sequence of method names found within testCaseClass - """ - - testFnNames = super(RegexTestLoader, self).getTestCaseNames(testCaseClass) - if self.regex is None: - return testFnNames - robj = re.compile(self.regex) - matched = [] - for attrname in testFnNames: - fullname = _format_test_name(testCaseClass, attrname) - if robj.search(fullname): - matched.append(attrname) - return matched - - -class Loader(object): - - suiteClass = unittest.TestSuite - - def __init__(self, modules, regex=None): - self.modules = modules - self.regex = regex - - def load_all(self): - loader = RegexTestLoader(self.regex) - test_suite = self.suiteClass() - for module_name in self.modules: - __import__(module_name, level=0) - module = sys.modules[module_name] - module_suite = loader.loadTestsFromModule(module) - test_suite.addTest(module_suite) - return test_suite - - def load_args(self, args): - loader = RegexTestLoader(self.regex) - - suites = [] - for arg in args: - suite = loader.loadTestsFromName(arg) - # loadTestsFromName() can only process names that refer to - # individual test functions or modules. It can't process package - # names. If there were no module/function matches, check to see if - # this looks like a package name. - if suite.countTestCases() != 0: - suites.append(suite) - continue - - # Load all modules whose name is . - prefix = arg + "." - for module in self.modules: - if module.startswith(prefix): - suite = loader.loadTestsFromName(module) - suites.append(suite) - - return loader.suiteClass(suites) - - -_COVERAGE_INI = """\ -[report] -exclude_lines = - pragma: no cover - pragma: nocover - pragma:.*no${PLATFORM} - pragma:.*no${PY_IMPL}${PY_MAJOR}${PY_MINOR} - pragma:.*no${PY_IMPL}${PY_MAJOR} - pragma:.*nopy${PY_MAJOR} - pragma:.*nopy${PY_MAJOR}${PY_MINOR} -""" - - -class MainProgram(object): - """ - This class implements the main program. It can be subclassed by - users who wish to customize some parts of the main program. - (Adding additional command line options, customizing test loading, etc.) - """ - - DEFAULT_VERBOSITY = 2 - - def __init__(self, argv): - self.init_option_parser() - self.parse_options(argv) - self.setup_logging() - - def init_option_parser(self): - usage = "%prog [options] [TEST] ..." - op = optparse.OptionParser(usage=usage, add_help_option=False) - self.option_parser = op - - op.add_option( - "--hide-output", - dest="show_output", - action="store_false", - default=True, - help="Suppress data that tests print to stdout/stderr, and only " - "show it if the test fails.", - ) - op.add_option( - "-o", - "--output", - help="Write results to a file in a JSON format to be read by Buck", - ) - op.add_option( - "-f", - "--failfast", - action="store_true", - default=False, - help="Stop after the first failure", - ) - op.add_option( - "-l", - "--list-tests", - action="store_true", - dest="list", - default=False, - help="List tests and exit", - ) - op.add_option( - "-r", - "--regex", - default=None, - help="Regex to apply to tests, to only run those tests", - ) - op.add_option( - "--collect-coverage", - action="store_true", - default=False, - help="Collect test coverage information", - ) - op.add_option( - "--coverage-include", - default="*", - help='File globs to include in converage (split by ",")', - ) - op.add_option( - "--coverage-omit", - default="", - help='File globs to omit from converage (split by ",")', - ) - op.add_option( - "--logger", - action="append", - metavar="=", - default=[], - help="Configure log levels for specific logger categories", - ) - op.add_option( - "-q", - "--quiet", - action="count", - default=0, - help="Decrease the verbosity (may be specified multiple times)", - ) - op.add_option( - "-v", - "--verbosity", - action="count", - default=self.DEFAULT_VERBOSITY, - help="Increase the verbosity (may be specified multiple times)", - ) - op.add_option( - "-?", "--help", action="help", help="Show this help message and exit" - ) - - def parse_options(self, argv): - self.options, self.test_args = self.option_parser.parse_args(argv[1:]) - self.options.verbosity -= self.options.quiet - - if self.options.collect_coverage and coverage is None: - self.option_parser.error("coverage module is not available") - self.options.coverage_include = self.options.coverage_include.split(",") - if self.options.coverage_omit == "": - self.options.coverage_omit = [] - else: - self.options.coverage_omit = self.options.coverage_omit.split(",") - - def setup_logging(self): - # Configure the root logger to log at INFO level. - # This is similar to logging.basicConfig(), but uses our - # StderrLogHandler instead of a StreamHandler. - fmt = logging.Formatter("%(pathname)s:%(lineno)s: %(message)s") - log_handler = StderrLogHandler() - log_handler.setFormatter(fmt) - root_logger = logging.getLogger() - root_logger.addHandler(log_handler) - root_logger.setLevel(logging.INFO) - - level_names = { - "debug": logging.DEBUG, - "info": logging.INFO, - "warn": logging.WARNING, - "warning": logging.WARNING, - "error": logging.ERROR, - "critical": logging.CRITICAL, - "fatal": logging.FATAL, - } - - for value in self.options.logger: - parts = value.rsplit("=", 1) - if len(parts) != 2: - self.option_parser.error( - "--logger argument must be of the " - "form =: %s" % value - ) - name = parts[0] - level_name = parts[1].lower() - level = level_names.get(level_name) - if level is None: - self.option_parser.error( - "invalid log level %r for log " "category %s" % (parts[1], name) - ) - logging.getLogger(name).setLevel(level) - - def create_loader(self): - import __test_modules__ - - return Loader(__test_modules__.TEST_MODULES, self.options.regex) - - def load_tests(self): - loader = self.create_loader() - if self.options.collect_coverage: - self.start_coverage() - include = self.options.coverage_include - omit = self.options.coverage_omit - if include and "*" not in include: - optimize_for_coverage(self.cov, include, omit) - - if self.test_args: - suite = loader.load_args(self.test_args) - else: - suite = loader.load_all() - if self.options.collect_coverage: - self.cov.start() - return suite - - def get_tests(self, test_suite): - tests = [] - - for test in test_suite: - if isinstance(test, unittest.TestSuite): - tests.extend(self.get_tests(test)) - else: - tests.append(test) - - return tests - - def run(self): - test_suite = self.load_tests() - - if self.options.list: - for test in self.get_tests(test_suite): - method_name = getattr(test, "_testMethodName", "") - name = _format_test_name(test.__class__, method_name) - print(name) - return EXIT_CODE_SUCCESS - else: - result = self.run_tests(test_suite) - if self.options.output is not None: - with open(self.options.output, "w") as f: - json.dump(result.getResults(), f, indent=4, sort_keys=True) - if not result.wasSuccessful(): - return EXIT_CODE_TEST_FAILURE - return EXIT_CODE_SUCCESS - - def run_tests(self, test_suite): - # Install a signal handler to catch Ctrl-C and display the results - # (but only if running >2.6). - if sys.version_info[0] > 2 or sys.version_info[1] > 6: - unittest.installHandler() - - # Run the tests - runner = BuckTestRunner( - self, - test_suite, - verbosity=self.options.verbosity, - show_output=self.options.show_output, - ) - result = runner.run(test_suite) - - if self.options.collect_coverage and self.options.show_output: - self.cov.stop() - try: - self.cov.report(file=sys.stdout) - except coverage.misc.CoverageException: - print("No lines were covered, potentially restricted by file filters") - - return result - - def get_abbr_impl(self): - """Return abbreviated implementation name.""" - impl = platform.python_implementation() - if impl == "PyPy": - return "pp" - elif impl == "Jython": - return "jy" - elif impl == "IronPython": - return "ip" - elif impl == "CPython": - return "cp" - else: - raise RuntimeError("unknown python runtime") - - def start_coverage(self): - if not self.options.collect_coverage: - return - - with tempfile.NamedTemporaryFile("w", delete=False) as coverage_ini: - coverage_ini.write(_COVERAGE_INI) - self._coverage_ini_path = coverage_ini.name - - # Keep the original working dir in case tests use os.chdir - self._original_working_dir = os.getcwd() - - # for coverage config ignores by platform/python version - os.environ["PLATFORM"] = sys.platform - os.environ["PY_IMPL"] = self.get_abbr_impl() - os.environ["PY_MAJOR"] = str(sys.version_info.major) - os.environ["PY_MINOR"] = str(sys.version_info.minor) - - self.cov = coverage.Coverage( - include=self.options.coverage_include, - omit=self.options.coverage_omit, - config_file=coverage_ini.name, - ) - self.cov.erase() - self.cov.start() - - def get_coverage(self): - if not self.options.collect_coverage: - return None - - try: - os.remove(self._coverage_ini_path) - except OSError: - pass # Better to litter than to fail the test - - # Switch back to the original working directory. - os.chdir(self._original_working_dir) - - result = {} - - self.cov.stop() - - try: - f = StringIO() - self.cov.report(file=f) - lines = f.getvalue().split("\n") - except coverage.misc.CoverageException: - # Nothing was covered. That's fine by us - return result - - # N.B.: the format of the coverage library's output differs - # depending on whether one or more files are in the results - for line in lines[2:]: - if line.strip("-") == "": - break - r = line.split()[0] - analysis = self.cov.analysis2(r) - covString = self.convert_to_diff_cov_str(analysis) - if covString: - result[r] = covString - - return result - - def convert_to_diff_cov_str(self, analysis): - # Info on the format of analysis: - # http://nedbatchelder.com/code/coverage/api.html - if not analysis: - return None - numLines = max( - analysis[1][-1] if len(analysis[1]) else 0, - analysis[2][-1] if len(analysis[2]) else 0, - analysis[3][-1] if len(analysis[3]) else 0, - ) - lines = ["N"] * numLines - for l in analysis[1]: - lines[l - 1] = "C" - for l in analysis[2]: - lines[l - 1] = "X" - for l in analysis[3]: - lines[l - 1] = "U" - return "".join(lines) - - -def main(argv): - return MainProgram(sys.argv).run() - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/build/fbcode_builder/CMake/fb_py_win_main.c b/build/fbcode_builder/CMake/fb_py_win_main.c deleted file mode 100644 index 21ee269be..000000000 --- a/build/fbcode_builder/CMake/fb_py_win_main.c +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (c) Facebook, Inc. and its affiliates. - -#define WIN32_LEAN_AND_MEAN - -#include -#include -#include - -#define PATH_SIZE 32768 - -typedef int (*Py_Main)(int, wchar_t**); - -int locate_py_main(int argc, wchar_t** argv) { - /* - * We have to dynamically locate Python3.dll because we may be loading a - * Python native module while running. If that module is built with a - * different Python version, we will end up a DLL import error. To resolve - * this, we can either ship an embedded version of Python with us or - * dynamically look up existing Python distribution installed on user's - * machine. This way, we should be able to get a consistent version of - * Python3.dll and .pyd modules. - */ - HINSTANCE python_dll; - Py_Main pymain; - - python_dll = - LoadLibraryExW(L"python3.dll", NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); - - int returncode = 0; - if (python_dll != NULL) { - pymain = (Py_Main)GetProcAddress(python_dll, "Py_Main"); - - if (pymain != NULL) { - returncode = (pymain)(argc, argv); - } else { - fprintf(stderr, "error: %d unable to load Py_Main\n", GetLastError()); - } - - FreeLibrary(python_dll); - } else { - fprintf(stderr, "error: %d unable to locate python3.dll\n", GetLastError()); - return 1; - } - return returncode; -} - -int wmain() { - /* - * This executable will be prepended to the start of a Python ZIP archive. - * Python will be able to directly execute the ZIP archive, so we simply - * need to tell Py_Main() to run our own file. Duplicate the argument list - * and add our file name to the beginning to tell Python what file to invoke. - */ - wchar_t** pyargv = malloc(sizeof(wchar_t*) * (__argc + 1)); - if (!pyargv) { - fprintf(stderr, "error: failed to allocate argument vector\n"); - return 1; - } - - /* Py_Main wants the wide character version of the argv so we pull those - * values from the global __wargv array that has been prepared by MSVCRT. - * - * In order for the zipapp to run we need to insert an extra argument in - * the front of the argument vector that points to ourselves. - * - * An additional complication is that, depending on who prepared the argument - * string used to start our process, the computed __wargv[0] can be a simple - * shell word like `watchman-wait` which is normally resolved together with - * the PATH by the shell. - * That unresolved path isn't sufficient to start the zipapp on windows; - * we need the fully qualified path. - * - * Given: - * __wargv == {"watchman-wait", "-h"} - * - * we want to pass the following to Py_Main: - * - * { - * "z:\build\watchman\python\watchman-wait.exe", - * "z:\build\watchman\python\watchman-wait.exe", - * "-h" - * } - */ - wchar_t full_path_to_argv0[PATH_SIZE]; - DWORD len = GetModuleFileNameW(NULL, full_path_to_argv0, PATH_SIZE); - if (len == 0 || - len == PATH_SIZE && GetLastError() == ERROR_INSUFFICIENT_BUFFER) { - fprintf( - stderr, - "error: %d while retrieving full path to this executable\n", - GetLastError()); - return 1; - } - - for (int n = 1; n < __argc; ++n) { - pyargv[n + 1] = __wargv[n]; - } - pyargv[0] = full_path_to_argv0; - pyargv[1] = full_path_to_argv0; - - return locate_py_main(__argc + 1, pyargv); -} diff --git a/build/fbcode_builder/CMake/make_fbpy_archive.py b/build/fbcode_builder/CMake/make_fbpy_archive.py deleted file mode 100755 index 70d3c426e..000000000 --- a/build/fbcode_builder/CMake/make_fbpy_archive.py +++ /dev/null @@ -1,359 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (c) Facebook, Inc. and its affiliates. -# -import argparse -import collections -import errno -import os -import shutil -import subprocess -import sys -import tempfile -import zipapp - -MANIFEST_SEPARATOR = " :: " -MANIFEST_HEADER_V1 = "FBPY_MANIFEST 1\n" - - -class UsageError(Exception): - def __init__(self, message): - self.message = message - - def __str__(self): - return self.message - - -class BadManifestError(UsageError): - def __init__(self, path, line_num, message): - full_msg = "%s:%s: %s" % (path, line_num, message) - super().__init__(full_msg) - self.path = path - self.line_num = line_num - self.raw_message = message - - -PathInfo = collections.namedtuple( - "PathInfo", ("src", "dest", "manifest_path", "manifest_line") -) - - -def parse_manifest(manifest, path_map): - bad_prefix = ".." + os.path.sep - manifest_dir = os.path.dirname(manifest) - with open(manifest, "r") as f: - line_num = 1 - line = f.readline() - if line != MANIFEST_HEADER_V1: - raise BadManifestError( - manifest, line_num, "Unexpected manifest file header" - ) - - for line in f: - line_num += 1 - if line.startswith("#"): - continue - line = line.rstrip("\n") - parts = line.split(MANIFEST_SEPARATOR) - if len(parts) != 2: - msg = "line must be of the form SRC %s DEST" % MANIFEST_SEPARATOR - raise BadManifestError(manifest, line_num, msg) - src, dest = parts - dest = os.path.normpath(dest) - if dest.startswith(bad_prefix): - msg = "destination path starts with %s: %s" % (bad_prefix, dest) - raise BadManifestError(manifest, line_num, msg) - - if not os.path.isabs(src): - src = os.path.normpath(os.path.join(manifest_dir, src)) - - if dest in path_map: - prev_info = path_map[dest] - msg = ( - "multiple source paths specified for destination " - "path %s. Previous source was %s from %s:%s" - % ( - dest, - prev_info.src, - prev_info.manifest_path, - prev_info.manifest_line, - ) - ) - raise BadManifestError(manifest, line_num, msg) - - info = PathInfo( - src=src, - dest=dest, - manifest_path=manifest, - manifest_line=line_num, - ) - path_map[dest] = info - - -def populate_install_tree(inst_dir, path_map): - os.mkdir(inst_dir) - dest_dirs = {"": False} - - def make_dest_dir(path): - if path in dest_dirs: - return - parent = os.path.dirname(path) - make_dest_dir(parent) - abs_path = os.path.join(inst_dir, path) - os.mkdir(abs_path) - dest_dirs[path] = False - - def install_file(info): - dir_name, base_name = os.path.split(info.dest) - make_dest_dir(dir_name) - if base_name == "__init__.py": - dest_dirs[dir_name] = True - abs_dest = os.path.join(inst_dir, info.dest) - shutil.copy2(info.src, abs_dest) - - # Copy all of the destination files - for info in path_map.values(): - install_file(info) - - # Create __init__ files in any directories that don't have them. - for dir_path, has_init in dest_dirs.items(): - if has_init: - continue - init_path = os.path.join(inst_dir, dir_path, "__init__.py") - with open(init_path, "w"): - pass - - -def build_pex(args, path_map): - """Create a self executing python binary using the PEX tool - - This type of Python binary is more complex as it requires a third-party tool, - but it does support native language extensions (.so/.dll files). - """ - dest_dir = os.path.dirname(args.output) - with tempfile.TemporaryDirectory(prefix="make_fbpy.", dir=dest_dir) as tmpdir: - inst_dir = os.path.join(tmpdir, "tree") - populate_install_tree(inst_dir, path_map) - - if os.path.exists(os.path.join(inst_dir, "__main__.py")): - os.rename( - os.path.join(inst_dir, "__main__.py"), - os.path.join(inst_dir, "main.py"), - ) - args.main = "main" - - tmp_output = os.path.abspath(os.path.join(tmpdir, "output.exe")) - subprocess.check_call( - ["pex"] - + ["--output-file", tmp_output] - + ["--python", args.python] - + ["--sources-directory", inst_dir] - + ["-e", args.main] - ) - - os.replace(tmp_output, args.output) - - -def build_zipapp(args, path_map): - """Create a self executing python binary using Python 3's built-in - zipapp module. - - This type of Python binary is relatively simple, as zipapp is part of the - standard library, but it does not support native language extensions - (.so/.dll files). - """ - dest_dir = os.path.dirname(args.output) - with tempfile.TemporaryDirectory(prefix="make_fbpy.", dir=dest_dir) as tmpdir: - inst_dir = os.path.join(tmpdir, "tree") - populate_install_tree(inst_dir, path_map) - - tmp_output = os.path.join(tmpdir, "output.exe") - zipapp.create_archive( - inst_dir, target=tmp_output, interpreter=args.python, main=args.main - ) - os.replace(tmp_output, args.output) - - -def create_main_module(args, inst_dir, path_map): - if not args.main: - assert "__main__.py" in path_map - return - - dest_path = os.path.join(inst_dir, "__main__.py") - main_module, main_fn = args.main.split(":") - main_contents = """\ -#!{python} - -if __name__ == "__main__": - import {main_module} - {main_module}.{main_fn}() -""".format( - python=args.python, main_module=main_module, main_fn=main_fn - ) - with open(dest_path, "w") as f: - f.write(main_contents) - os.chmod(dest_path, 0o755) - - -def build_install_dir(args, path_map): - """Create a directory that contains all of the sources, with a __main__ - module to run the program. - """ - # Populate a temporary directory first, then rename to the destination - # location. This ensures that we don't ever leave a halfway-built - # directory behind at the output path if something goes wrong. - dest_dir = os.path.dirname(args.output) - with tempfile.TemporaryDirectory(prefix="make_fbpy.", dir=dest_dir) as tmpdir: - inst_dir = os.path.join(tmpdir, "tree") - populate_install_tree(inst_dir, path_map) - create_main_module(args, inst_dir, path_map) - os.rename(inst_dir, args.output) - - -def ensure_directory(path): - try: - os.makedirs(path) - except OSError as ex: - if ex.errno != errno.EEXIST: - raise - - -def install_library(args, path_map): - """Create an installation directory a python library.""" - out_dir = args.output - out_manifest = args.output + ".manifest" - - install_dir = args.install_dir - if not install_dir: - install_dir = out_dir - - os.makedirs(out_dir) - with open(out_manifest, "w") as manifest: - manifest.write(MANIFEST_HEADER_V1) - for info in path_map.values(): - abs_dest = os.path.join(out_dir, info.dest) - ensure_directory(os.path.dirname(abs_dest)) - print("copy %r --> %r" % (info.src, abs_dest)) - shutil.copy2(info.src, abs_dest) - installed_dest = os.path.join(install_dir, info.dest) - manifest.write("%s%s%s\n" % (installed_dest, MANIFEST_SEPARATOR, info.dest)) - - -def parse_manifests(args): - # Process args.manifest_separator to help support older versions of CMake - if args.manifest_separator: - manifests = [] - for manifest_arg in args.manifests: - split_arg = manifest_arg.split(args.manifest_separator) - manifests.extend(split_arg) - args.manifests = manifests - - path_map = {} - for manifest in args.manifests: - parse_manifest(manifest, path_map) - - return path_map - - -def check_main_module(args, path_map): - # Translate an empty string in the --main argument to None, - # just to allow the CMake logic to be slightly simpler and pass in an - # empty string when it really wants the default __main__.py module to be - # used. - if args.main == "": - args.main = None - - if args.type == "lib-install": - if args.main is not None: - raise UsageError("cannot specify a --main argument with --type=lib-install") - return - - main_info = path_map.get("__main__.py") - if args.main: - if main_info is not None: - msg = ( - "specified an explicit main module with --main, " - "but the file listing already includes __main__.py" - ) - raise BadManifestError( - main_info.manifest_path, main_info.manifest_line, msg - ) - parts = args.main.split(":") - if len(parts) != 2: - raise UsageError( - "argument to --main must be of the form MODULE:CALLABLE " - "(received %s)" % (args.main,) - ) - else: - if main_info is None: - raise UsageError( - "no main module specified with --main, " - "and no __main__.py module present" - ) - - -BUILD_TYPES = { - "pex": build_pex, - "zipapp": build_zipapp, - "dir": build_install_dir, - "lib-install": install_library, -} - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("-o", "--output", required=True, help="The output file path") - ap.add_argument( - "--install-dir", - help="When used with --type=lib-install, this parameter specifies the " - "final location where the library where be installed. This can be " - "used to generate the library in one directory first, when you plan " - "to move or copy it to another final location later.", - ) - ap.add_argument( - "--manifest-separator", - help="Split manifest arguments around this separator. This is used " - "to support older versions of CMake that cannot supply the manifests " - "as separate arguments.", - ) - ap.add_argument( - "--main", - help="The main module to run, specified as :. " - "This must be specified if and only if the archive does not contain " - "a __main__.py file.", - ) - ap.add_argument( - "--python", - help="Explicitly specify the python interpreter to use for the " "executable.", - ) - ap.add_argument( - "--type", choices=BUILD_TYPES.keys(), help="The type of output to build." - ) - ap.add_argument( - "manifests", - nargs="+", - help="The manifest files specifying how to construct the archive", - ) - args = ap.parse_args() - - if args.python is None: - args.python = sys.executable - - if args.type is None: - # In the future we might want different default output types - # for different platforms. - args.type = "zipapp" - build_fn = BUILD_TYPES[args.type] - - try: - path_map = parse_manifests(args) - check_main_module(args, path_map) - except UsageError as ex: - print("error: %s" % (ex,), file=sys.stderr) - sys.exit(1) - - build_fn(args, path_map) - - -if __name__ == "__main__": - main() diff --git a/build/fbcode_builder/LICENSE b/build/fbcode_builder/LICENSE deleted file mode 100644 index b96dcb048..000000000 --- a/build/fbcode_builder/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) Facebook, Inc. and its affiliates. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/build/fbcode_builder/README.md b/build/fbcode_builder/README.md deleted file mode 100644 index d47dd41c0..000000000 --- a/build/fbcode_builder/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Easy builds for Facebook projects - -This directory contains tools designed to simplify continuous-integration -(and other builds) of Facebook open source projects. In particular, this helps -manage builds for cross-project dependencies. - -The main entry point is the `getdeps.py` script. This script has several -subcommands, but the most notable is the `build` command. This will download -and build all dependencies for a project, and then build the project itself. - -## Deployment - -This directory is copied literally into a number of different Facebook open -source repositories. Any change made to code in this directory will be -automatically be replicated by our open source tooling into all GitHub hosted -repositories that use `fbcode_builder`. Typically this directory is copied -into the open source repositories as `build/fbcode_builder/`. - - -# Project Configuration Files - -The `manifests` subdirectory contains configuration files for many different -projects, describing how to build each project. These files also list -dependencies between projects, enabling `getdeps.py` to build all dependencies -for a project before building the project itself. - - -# Shared CMake utilities - -Since this directory is copied into many Facebook open source repositories, -it is also used to help share some CMake utility files across projects. The -`CMake/` subdirectory contains a number of `.cmake` files that are shared by -the CMake-based build systems across several different projects. - - -# Older Build Scripts - -This directory also still contains a handful of older build scripts that -pre-date the current `getdeps.py` build system. Most of the other `.py` files -in this top directory, apart from `getdeps.py` itself, are from this older -build system. This older system is only used by a few remaining projects, and -new projects should generally use the newer `getdeps.py` script, by adding a -new configuration file in the `manifests/` subdirectory. diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py deleted file mode 100755 index cf29ccab8..000000000 --- a/build/fbcode_builder/getdeps.py +++ /dev/null @@ -1,1632 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import argparse -import json -import os -import shutil -import subprocess -import sys -import tarfile -import tempfile - -# We don't import cache.create_cache directly as the facebook -# specific import below may monkey patch it, and we want to -# observe the patched version of this function! -import getdeps.cache as cache_module -from getdeps.buildopts import setup_build_options -from getdeps.dyndeps import create_dyn_dep_munger -from getdeps.errors import TransientFailure -from getdeps.fetcher import ( - file_name_is_cmake_file, - is_public_commit, - list_files_under_dir_newer_than_timestamp, - SystemPackageFetcher, -) -from getdeps.load import ManifestLoader -from getdeps.manifest import ManifestParser -from getdeps.platform import HostType -from getdeps.runcmd import check_cmd -from getdeps.subcmd import add_subcommands, cmd, SubCmd - -try: - import getdeps.facebook # noqa: F401 -except ImportError: - # we don't ship the facebook specific subdir, - # so allow that to fail silently - pass - - -sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "getdeps")) - - -class UsageError(Exception): - pass - - -@cmd("validate-manifest", "parse a manifest and validate that it is correct") -class ValidateManifest(SubCmd): - def run(self, args): - try: - ManifestParser(file_name=args.file_name) - print("OK", file=sys.stderr) - return 0 - except Exception as exc: - print("ERROR: %s" % str(exc), file=sys.stderr) - return 1 - - def setup_parser(self, parser): - parser.add_argument("file_name", help="path to the manifest file") - - -@cmd("show-host-type", "outputs the host type tuple for the host machine") -class ShowHostType(SubCmd): - def run(self, args): - host = HostType() - print("%s" % host.as_tuple_string()) - return 0 - - -class ProjectCmdBase(SubCmd): - def run(self, args): - opts = setup_build_options(args) - - if args.current_project is not None: - opts.repo_project = args.current_project - if args.project is None: - if opts.repo_project is None: - raise UsageError( - "no project name specified, and no .projectid file found" - ) - if opts.repo_project == "fbsource": - # The fbsource repository is a little special. There is no project - # manifest file for it. A specific project must always be explicitly - # specified when building from fbsource. - raise UsageError( - "no project name specified (required when building in fbsource)" - ) - args.project = opts.repo_project - - ctx_gen = opts.get_context_generator() - if args.test_dependencies: - ctx_gen.set_value_for_all_projects("test", "on") - if args.enable_tests: - ctx_gen.set_value_for_project(args.project, "test", "on") - else: - ctx_gen.set_value_for_project(args.project, "test", "off") - - if opts.shared_libs: - ctx_gen.set_value_for_all_projects("shared_libs", "on") - - loader = ManifestLoader(opts, ctx_gen) - self.process_project_dir_arguments(args, loader) - - manifest = loader.load_manifest(args.project) - - return self.run_project_cmd(args, loader, manifest) - - def process_project_dir_arguments(self, args, loader): - def parse_project_arg(arg, arg_type): - parts = arg.split(":") - if len(parts) == 2: - project, path = parts - elif len(parts) == 1: - project = args.project - path = parts[0] - # On Windows path contains colon, e.g. C:\open - elif os.name == "nt" and len(parts) == 3: - project = parts[0] - path = parts[1] + ":" + parts[2] - else: - raise UsageError( - "invalid %s argument; too many ':' characters: %s" % (arg_type, arg) - ) - - return project, os.path.abspath(path) - - # If we are currently running from a project repository, - # use the current repository for the project sources. - build_opts = loader.build_opts - if build_opts.repo_project is not None and build_opts.repo_root is not None: - loader.set_project_src_dir(build_opts.repo_project, build_opts.repo_root) - - for arg in args.src_dir: - project, path = parse_project_arg(arg, "--src-dir") - loader.set_project_src_dir(project, path) - - for arg in args.build_dir: - project, path = parse_project_arg(arg, "--build-dir") - loader.set_project_build_dir(project, path) - - for arg in args.install_dir: - project, path = parse_project_arg(arg, "--install-dir") - loader.set_project_install_dir(project, path) - - for arg in args.project_install_prefix: - project, path = parse_project_arg(arg, "--install-prefix") - loader.set_project_install_prefix(project, path) - - def setup_parser(self, parser): - parser.add_argument( - "project", - nargs="?", - help=( - "name of the project or path to a manifest " - "file describing the project" - ), - ) - parser.add_argument( - "--no-tests", - action="store_false", - dest="enable_tests", - default=True, - help="Disable building tests for this project.", - ) - parser.add_argument( - "--test-dependencies", - action="store_true", - help="Enable building tests for dependencies as well.", - ) - parser.add_argument( - "--current-project", - help="Specify the name of the fbcode_builder manifest file for the " - "current repository. If not specified, the code will attempt to find " - "this in a .projectid file in the repository root.", - ) - parser.add_argument( - "--src-dir", - default=[], - action="append", - help="Specify a local directory to use for the project source, " - "rather than fetching it.", - ) - parser.add_argument( - "--build-dir", - default=[], - action="append", - help="Explicitly specify the build directory to use for the " - "project, instead of the default location in the scratch path. " - "This only affects the project specified, and not its dependencies.", - ) - parser.add_argument( - "--install-dir", - default=[], - action="append", - help="Explicitly specify the install directory to use for the " - "project, instead of the default location in the scratch path. " - "This only affects the project specified, and not its dependencies.", - ) - parser.add_argument( - "--project-install-prefix", - default=[], - action="append", - help="Specify the final deployment installation path for a project", - ) - - self.setup_project_cmd_parser(parser) - - def setup_project_cmd_parser(self, parser): - pass - - def create_builder(self, loader, manifest): - fetcher = loader.create_fetcher(manifest) - src_dir = fetcher.get_src_dir() - ctx = loader.ctx_gen.get_context(manifest.name) - build_dir = loader.get_project_build_dir(manifest) - inst_dir = loader.get_project_install_dir(manifest) - return manifest.create_builder( - loader.build_opts, - src_dir, - build_dir, - inst_dir, - ctx, - loader, - loader.dependencies_of(manifest), - ) - - def check_built(self, loader, manifest): - built_marker = os.path.join( - loader.get_project_install_dir(manifest), ".built-by-getdeps" - ) - return os.path.exists(built_marker) - - -class CachedProject(object): - """A helper that allows calling the cache logic for a project - from both the build and the fetch code""" - - def __init__(self, cache, loader, m): - self.m = m - self.inst_dir = loader.get_project_install_dir(m) - self.project_hash = loader.get_project_hash(m) - self.ctx = loader.ctx_gen.get_context(m.name) - self.loader = loader - self.cache = cache - - self.cache_key = "-".join( - ( - m.name, - self.ctx.get("os"), - self.ctx.get("distro") or "none", - self.ctx.get("distro_vers") or "none", - self.project_hash, - ) - ) - self.cache_file_name = self.cache_key + "-buildcache.tgz" - - def is_cacheable(self): - """We only cache third party projects""" - return self.cache and self.m.shipit_project is None - - def was_cached(self): - cached_marker = os.path.join(self.inst_dir, ".getdeps-cached-build") - return os.path.exists(cached_marker) - - def download(self): - if self.is_cacheable() and not os.path.exists(self.inst_dir): - print("check cache for %s" % self.cache_file_name) - dl_dir = os.path.join(self.loader.build_opts.scratch_dir, "downloads") - if not os.path.exists(dl_dir): - os.makedirs(dl_dir) - try: - target_file_name = os.path.join(dl_dir, self.cache_file_name) - if self.cache.download_to_file(self.cache_file_name, target_file_name): - tf = tarfile.open(target_file_name, "r") - print( - "Extracting %s -> %s..." % (self.cache_file_name, self.inst_dir) - ) - tf.extractall(self.inst_dir) - - cached_marker = os.path.join(self.inst_dir, ".getdeps-cached-build") - with open(cached_marker, "w") as f: - f.write("\n") - - return True - except Exception as exc: - print("%s" % str(exc)) - - return False - - def upload(self): - if self.is_cacheable(): - # We can prepare an archive and stick it in LFS - tempdir = tempfile.mkdtemp() - tarfilename = os.path.join(tempdir, self.cache_file_name) - print("Archiving for cache: %s..." % tarfilename) - tf = tarfile.open(tarfilename, "w:gz") - tf.add(self.inst_dir, arcname=".") - tf.close() - try: - self.cache.upload_from_file(self.cache_file_name, tarfilename) - except Exception as exc: - print( - "Failed to upload to cache (%s), continue anyway" % str(exc), - file=sys.stderr, - ) - shutil.rmtree(tempdir) - - -@cmd("fetch", "fetch the code for a given project") -class FetchCmd(ProjectCmdBase): - def setup_project_cmd_parser(self, parser): - parser.add_argument( - "--recursive", - help="fetch the transitive deps also", - action="store_true", - default=False, - ) - parser.add_argument( - "--host-type", - help=( - "When recursively fetching, fetch deps for " - "this host type rather than the current system" - ), - ) - - def run_project_cmd(self, args, loader, manifest): - if args.recursive: - projects = loader.manifests_in_dependency_order() - else: - projects = [manifest] - - cache = cache_module.create_cache() - for m in projects: - fetcher = loader.create_fetcher(m) - if isinstance(fetcher, SystemPackageFetcher): - # We are guaranteed that if the fetcher is set to - # SystemPackageFetcher then this item is completely - # satisfied by the appropriate system packages - continue - cached_project = CachedProject(cache, loader, m) - if cached_project.download(): - continue - - inst_dir = loader.get_project_install_dir(m) - built_marker = os.path.join(inst_dir, ".built-by-getdeps") - if os.path.exists(built_marker): - with open(built_marker, "r") as f: - built_hash = f.read().strip() - - project_hash = loader.get_project_hash(m) - if built_hash == project_hash: - continue - - # We need to fetch the sources - fetcher.update() - - -@cmd("install-system-deps", "Install system packages to satisfy the deps for a project") -class InstallSysDepsCmd(ProjectCmdBase): - def setup_project_cmd_parser(self, parser): - parser.add_argument( - "--recursive", - help="install the transitive deps also", - action="store_true", - default=False, - ) - parser.add_argument( - "--dry-run", - action="store_true", - default=False, - help="Don't install, just print the commands specs we would run", - ) - parser.add_argument( - "--os-type", - help="Filter to just this OS type to run", - choices=["linux", "darwin", "windows", "pacman-package"], - action="store", - dest="ostype", - default=None, - ) - parser.add_argument( - "--distro", - help="Filter to just this distro to run", - choices=["ubuntu", "centos_stream"], - action="store", - dest="distro", - default=None, - ) - parser.add_argument( - "--distro-version", - help="Filter to just this distro version", - action="store", - dest="distrovers", - default=None, - ) - - def run_project_cmd(self, args, loader, manifest): - if args.recursive: - projects = loader.manifests_in_dependency_order() - else: - projects = [manifest] - - rebuild_ctx_gen = False - if args.ostype: - loader.build_opts.host_type.ostype = args.ostype - loader.build_opts.host_type.distro = None - loader.build_opts.host_type.distrovers = None - rebuild_ctx_gen = True - - if args.distro: - loader.build_opts.host_type.distro = args.distro - loader.build_opts.host_type.distrovers = None - rebuild_ctx_gen = True - - if args.distrovers: - loader.build_opts.host_type.distrovers = args.distrovers - rebuild_ctx_gen = True - - if rebuild_ctx_gen: - loader.ctx_gen = loader.build_opts.get_context_generator() - - manager = loader.build_opts.host_type.get_package_manager() - - all_packages = {} - for m in projects: - ctx = loader.ctx_gen.get_context(m.name) - packages = m.get_required_system_packages(ctx) - for k, v in packages.items(): - merged = all_packages.get(k, []) - merged += v - all_packages[k] = merged - - cmd_argss = [] - if manager == "rpm": - packages = sorted(set(all_packages["rpm"])) - if packages: - cmd_argss.append( - ["sudo", "dnf", "install", "-y", "--skip-broken"] + packages - ) - elif manager == "deb": - packages = sorted(set(all_packages["deb"])) - if packages: - cmd_argss.append( - [ - "sudo", - "--preserve-env=http_proxy", - "apt-get", - "install", - "-y", - ] - + packages - ) - cmd_argss.append(["pip", "install", "pex"]) - elif manager == "homebrew": - packages = sorted(set(all_packages["homebrew"])) - if packages: - cmd_argss.append(["brew", "install"] + packages) - elif manager == "pacman-package": - packages = sorted(list(set(all_packages["pacman-package"]))) - if packages: - cmd_argss.append(["pacman", "-S"] + packages) - else: - host_tuple = loader.build_opts.host_type.as_tuple_string() - print( - f"I don't know how to install any packages on this system {host_tuple}" - ) - return - - for cmd_args in cmd_argss: - if args.dry_run: - print(" ".join(cmd_args)) - else: - check_cmd(cmd_args) - else: - print("no packages to install") - - -@cmd("list-deps", "lists the transitive deps for a given project") -class ListDepsCmd(ProjectCmdBase): - def run_project_cmd(self, args, loader, manifest): - for m in loader.manifests_in_dependency_order(): - print(m.name) - return 0 - - def setup_project_cmd_parser(self, parser): - parser.add_argument( - "--host-type", - help=( - "Produce the list for the specified host type, " - "rather than that of the current system" - ), - ) - - -def clean_dirs(opts): - for d in ["build", "installed", "extracted", "shipit"]: - d = os.path.join(opts.scratch_dir, d) - print("Cleaning %s..." % d) - if os.path.exists(d): - shutil.rmtree(d) - - -@cmd("clean", "clean up the scratch dir") -class CleanCmd(SubCmd): - def run(self, args): - opts = setup_build_options(args) - clean_dirs(opts) - - -@cmd("show-scratch-dir", "show the scratch dir") -class ShowScratchDirCmd(SubCmd): - def run(self, args): - opts = setup_build_options(args) - print(opts.scratch_dir) - - -@cmd("show-build-dir", "print the build dir for a given project") -class ShowBuildDirCmd(ProjectCmdBase): - def run_project_cmd(self, args, loader, manifest): - if args.recursive: - manifests = loader.manifests_in_dependency_order() - else: - manifests = [manifest] - - for m in manifests: - inst_dir = loader.get_project_build_dir(m) - print(inst_dir) - - def setup_project_cmd_parser(self, parser): - parser.add_argument( - "--recursive", - help="print the transitive deps also", - action="store_true", - default=False, - ) - - -@cmd("show-inst-dir", "print the installation dir for a given project") -class ShowInstDirCmd(ProjectCmdBase): - def run_project_cmd(self, args, loader, manifest): - if args.recursive: - manifests = loader.manifests_in_dependency_order() - else: - manifests = [manifest] - - for m in manifests: - fetcher = loader.create_fetcher(m) - if isinstance(fetcher, SystemPackageFetcher): - # We are guaranteed that if the fetcher is set to - # SystemPackageFetcher then this item is completely - # satisfied by the appropriate system packages - continue - inst_dir = loader.get_project_install_dir_respecting_install_prefix(m) - print(inst_dir) - - def setup_project_cmd_parser(self, parser): - parser.add_argument( - "--recursive", - help="print the transitive deps also", - action="store_true", - default=False, - ) - - -@cmd("query-paths", "print the paths for tooling to use") -class QueryPathsCmd(ProjectCmdBase): - def run_project_cmd(self, args, loader, manifest): - if args.recursive: - manifests = loader.manifests_in_dependency_order() - else: - manifests = [manifest] - - cache = cache_module.create_cache() - for m in manifests: - fetcher = loader.create_fetcher(m) - if isinstance(fetcher, SystemPackageFetcher): - # We are guaranteed that if the fetcher is set to - # SystemPackageFetcher then this item is completely - # satisfied by the appropriate system packages - continue - src_dir = fetcher.get_src_dir() - print(f"{m.name}_SOURCE={src_dir}") - inst_dir = loader.get_project_install_dir_respecting_install_prefix(m) - print(f"{m.name}_INSTALL={inst_dir}") - cached_project = CachedProject(cache, loader, m) - print(f"{m.name}_CACHE_KEY={cached_project.cache_key}") - - def setup_project_cmd_parser(self, parser): - parser.add_argument( - "--recursive", - help="print the transitive deps also", - action="store_true", - default=False, - ) - - -@cmd("show-source-dir", "print the source dir for a given project") -class ShowSourceDirCmd(ProjectCmdBase): - def run_project_cmd(self, args, loader, manifest): - if args.recursive: - manifests = loader.manifests_in_dependency_order() - else: - manifests = [manifest] - - for m in manifests: - fetcher = loader.create_fetcher(m) - print(fetcher.get_src_dir()) - - def setup_project_cmd_parser(self, parser): - parser.add_argument( - "--recursive", - help="print the transitive deps also", - action="store_true", - default=False, - ) - - -@cmd("build", "build a given project") -class BuildCmd(ProjectCmdBase): - def run_project_cmd(self, args, loader, manifest): - if args.clean: - clean_dirs(loader.build_opts) - - print("Building on %s" % loader.ctx_gen.get_context(args.project)) - projects = loader.manifests_in_dependency_order() - - cache = cache_module.create_cache() if args.use_build_cache else None - - dep_manifests = [] - - for m in projects: - dep_manifests.append(m) - - fetcher = loader.create_fetcher(m) - - if args.build_skip_lfs_download and hasattr(fetcher, "skip_lfs_download"): - print("skipping lfs download for %s" % m.name) - fetcher.skip_lfs_download() - - if isinstance(fetcher, SystemPackageFetcher): - # We are guaranteed that if the fetcher is set to - # SystemPackageFetcher then this item is completely - # satisfied by the appropriate system packages - continue - - if args.clean: - fetcher.clean() - - build_dir = loader.get_project_build_dir(m) - inst_dir = loader.get_project_install_dir(m) - - if ( - m == manifest - and not args.only_deps - or m != manifest - and not args.no_deps - ): - print("Assessing %s..." % m.name) - project_hash = loader.get_project_hash(m) - ctx = loader.ctx_gen.get_context(m.name) - built_marker = os.path.join(inst_dir, ".built-by-getdeps") - - cached_project = CachedProject(cache, loader, m) - - reconfigure, sources_changed = self.compute_source_change_status( - cached_project, fetcher, m, built_marker, project_hash - ) - - if os.path.exists(built_marker) and not cached_project.was_cached(): - # We've previously built this. We may need to reconfigure if - # our deps have changed, so let's check them. - dep_reconfigure, dep_build = self.compute_dep_change_status( - m, built_marker, loader - ) - if dep_reconfigure: - reconfigure = True - if dep_build: - sources_changed = True - - extra_cmake_defines = ( - json.loads(args.extra_cmake_defines) - if args.extra_cmake_defines - else {} - ) - - extra_b2_args = args.extra_b2_args or [] - cmake_targets = args.cmake_target or ["install"] - - if sources_changed or reconfigure or not os.path.exists(built_marker): - if os.path.exists(built_marker): - os.unlink(built_marker) - src_dir = fetcher.get_src_dir() - # Prepare builders write out config before the main builder runs - prepare_builders = m.create_prepare_builders( - loader.build_opts, - ctx, - src_dir, - build_dir, - inst_dir, - loader, - dep_manifests, - ) - for preparer in prepare_builders: - preparer.prepare(reconfigure=reconfigure) - - builder = m.create_builder( - loader.build_opts, - src_dir, - build_dir, - inst_dir, - ctx, - loader, - dep_manifests, - final_install_prefix=loader.get_project_install_prefix(m), - extra_cmake_defines=extra_cmake_defines, - cmake_targets=(cmake_targets if m == manifest else ["install"]), - extra_b2_args=extra_b2_args, - ) - builder.build(reconfigure=reconfigure) - - # If we are building the project (not dependency) and a specific - # cmake_target (not 'install') has been requested, then we don't - # set the built_marker. This allows subsequent runs of getdeps.py - # for the project to run with different cmake_targets to trigger - # cmake - has_built_marker = False - if not (m == manifest and "install" not in cmake_targets): - with open(built_marker, "w") as f: - f.write(project_hash) - has_built_marker = True - - # Only populate the cache from continuous build runs, and - # only if we have a built_marker. - if not args.skip_upload and has_built_marker: - if args.schedule_type == "continuous": - cached_project.upload() - elif args.schedule_type == "base_retry": - # Check if on public commit before uploading - if is_public_commit(loader.build_opts): - cached_project.upload() - elif args.verbose: - print("found good %s" % built_marker) - - def compute_dep_change_status(self, m, built_marker, loader): - reconfigure = False - sources_changed = False - st = os.lstat(built_marker) - - ctx = loader.ctx_gen.get_context(m.name) - dep_list = m.get_dependencies(ctx) - for dep in dep_list: - if reconfigure and sources_changed: - break - - dep_manifest = loader.load_manifest(dep) - dep_root = loader.get_project_install_dir(dep_manifest) - for dep_file in list_files_under_dir_newer_than_timestamp( - dep_root, st.st_mtime - ): - if os.path.basename(dep_file) == ".built-by-getdeps": - continue - if file_name_is_cmake_file(dep_file): - if not reconfigure: - reconfigure = True - print( - f"Will reconfigure cmake because {dep_file} is newer than {built_marker}" - ) - else: - if not sources_changed: - sources_changed = True - print( - f"Will run build because {dep_file} is newer than {built_marker}" - ) - - if reconfigure and sources_changed: - break - - return reconfigure, sources_changed - - def compute_source_change_status( - self, cached_project, fetcher, m, built_marker, project_hash - ): - reconfigure = False - sources_changed = False - if cached_project.download(): - if not os.path.exists(built_marker): - fetcher.update() - else: - check_fetcher = True - if os.path.exists(built_marker): - check_fetcher = False - with open(built_marker, "r") as f: - built_hash = f.read().strip() - if built_hash == project_hash: - if cached_project.is_cacheable(): - # We can blindly trust the build status - reconfigure = False - sources_changed = False - else: - # Otherwise, we may have changed the source, so let's - # check in with the fetcher layer - check_fetcher = True - else: - # Some kind of inconsistency with a prior build, - # let's run it again to be sure - os.unlink(built_marker) - reconfigure = True - sources_changed = True - # While we don't need to consult the fetcher for the - # status in this case, we may still need to have eg: shipit - # run in order to have a correct source tree. - fetcher.update() - - if check_fetcher: - change_status = fetcher.update() - reconfigure = change_status.build_changed() - sources_changed = change_status.sources_changed() - - return reconfigure, sources_changed - - def setup_project_cmd_parser(self, parser): - parser.add_argument( - "--clean", - action="store_true", - default=False, - help=( - "Clean up the build and installation area prior to building, " - "causing the projects to be built from scratch" - ), - ) - parser.add_argument( - "--no-deps", - action="store_true", - default=False, - help=( - "Only build the named project, not its deps. " - "This is most useful after you've built all of the deps, " - "and helps to avoid waiting for relatively " - "slow up-to-date-ness checks" - ), - ) - parser.add_argument( - "--only-deps", - action="store_true", - default=False, - help=( - "Only build the named project's deps. " - "This is most useful when you want to separate out building " - "of all of the deps and your project" - ), - ) - parser.add_argument( - "--no-build-cache", - action="store_false", - default=True, - dest="use_build_cache", - help="Do not attempt to use the build cache.", - ) - parser.add_argument( - "--cmake-target", - help=("Repeatable argument that specifies targets for cmake build."), - default=[], - action="append", - ) - parser.add_argument( - "--extra-b2-args", - help=( - "Repeatable argument that contains extra arguments to pass " - "to b2, which compiles boost. " - "e.g.: 'cxxflags=-fPIC' 'cflags=-fPIC'" - ), - action="append", - ) - parser.add_argument( - "--free-up-disk", - help="Remove unused tools and clean up intermediate files if possible to maximise space for the build", - action="store_true", - default=False, - ) - parser.add_argument( - "--build-type", - help="Set the build type explicitly. Cmake and cargo builders act on them. Only Debug and RelWithDebInfo widely supported.", - choices=["Debug", "Release", "RelWithDebInfo", "MinSizeRel"], - action="store", - default=None, - ) - - -@cmd("fixup-dyn-deps", "Adjusts dynamic dependencies for packaging purposes") -class FixupDeps(ProjectCmdBase): - def run_project_cmd(self, args, loader, manifest): - projects = loader.manifests_in_dependency_order() - - # Accumulate the install directories so that the build steps - # can find their dep installation - install_dirs = [] - dep_manifests = [] - - for m in projects: - inst_dir = loader.get_project_install_dir_respecting_install_prefix(m) - install_dirs.append(inst_dir) - dep_manifests.append(m) - - if m == manifest: - ctx = loader.ctx_gen.get_context(m.name) - env = loader.build_opts.compute_env_for_install_dirs( - loader, dep_manifests, ctx - ) - dep_munger = create_dyn_dep_munger( - loader.build_opts, env, install_dirs, args.strip - ) - if dep_munger is None: - print(f"dynamic dependency fixups not supported on {sys.platform}") - else: - dep_munger.process_deps(args.destdir, args.final_install_prefix) - - def setup_project_cmd_parser(self, parser): - parser.add_argument("destdir", help="Where to copy the fixed up executables") - parser.add_argument( - "--final-install-prefix", help="specify the final installation prefix" - ) - parser.add_argument( - "--strip", - action="store_true", - default=False, - help="Strip debug info while processing executables", - ) - - -@cmd("test", "test a given project") -class TestCmd(ProjectCmdBase): - def run_project_cmd(self, args, loader, manifest): - if not self.check_built(loader, manifest): - print("project %s has not been built" % manifest.name) - return 1 - return self.create_builder(loader, manifest).run_tests( - schedule_type=args.schedule_type, - owner=args.test_owner, - test_filter=args.filter, - test_exclude=args.exclude, - retry=args.retry, - no_testpilot=args.no_testpilot, - timeout=args.timeout, - ) - - def setup_project_cmd_parser(self, parser): - parser.add_argument("--test-owner", help="Owner for testpilot") - parser.add_argument("--filter", help="Only run the tests matching the regex") - parser.add_argument("--exclude", help="Exclude tests matching the regex") - parser.add_argument( - "--retry", - type=int, - default=3, - help="Number of immediate retries for failed tests " - "(noop in continuous and testwarden runs)", - ) - parser.add_argument( - "--no-testpilot", - help="Do not use Test Pilot even when available", - action="store_true", - ) - parser.add_argument( - "--timeout", - type=int, - default=None, - help="Timeout in seconds for each individual test", - ) - parser.add_argument( - "--build-type", - help="Set the build type explicitly. Cmake and cargo builders act on them. Only Debug and RelWithDebInfo widely supported.", - choices=["Debug", "Release", "RelWithDebInfo", "MinSizeRel"], - action="store", - default=None, - ) - - -@cmd( - "debug", - "start a shell in the given project's build dir with the correct environment for running the build", -) -class DebugCmd(ProjectCmdBase): - def run_project_cmd(self, args, loader, manifest): - self.create_builder(loader, manifest).debug(reconfigure=False) - - -@cmd( - "env", - "print the environment in a shell sourceable format", -) -class EnvCmd(ProjectCmdBase): - def setup_project_cmd_parser(self, parser): - parser.add_argument( - "--os-type", - help="Filter to just this OS type to run", - choices=["linux", "darwin", "windows"], - action="store", - dest="ostype", - default=None, - ) - - def run_project_cmd(self, args, loader, manifest): - if args.ostype: - loader.build_opts.host_type.ostype = args.ostype - self.create_builder(loader, manifest).printenv(reconfigure=False) - - -@cmd("generate-github-actions", "generate a GitHub actions configuration") -class GenerateGitHubActionsCmd(ProjectCmdBase): - RUN_ON_ALL = """ [push, pull_request]""" - - def run_project_cmd(self, args, loader, manifest): - platforms = [ - HostType("linux", "ubuntu", "22"), - HostType("darwin", None, None), - HostType("windows", None, None), - ] - - for p in platforms: - if args.os_types and p.ostype not in args.os_types: - continue - self.write_job_for_platform(p, args) - - def get_run_on(self, args): - if args.run_on_all_branches: - return self.RUN_ON_ALL - if args.cron: - if args.cron == "never": - return " {}" - elif args.cron == "workflow_dispatch": - return "\n workflow_dispatch" - else: - return f""" - schedule: - - cron: '{args.cron}'""" - - return f""" - push: - branches: - - {args.main_branch} - pull_request: - branches: - - {args.main_branch}""" - - # TODO: Break up complex function - def write_job_for_platform(self, platform, args): # noqa: C901 - build_opts = setup_build_options(args, platform) - ctx_gen = build_opts.get_context_generator() - if args.enable_tests: - ctx_gen.set_value_for_project(args.project, "test", "on") - else: - ctx_gen.set_value_for_project(args.project, "test", "off") - loader = ManifestLoader(build_opts, ctx_gen) - self.process_project_dir_arguments(args, loader) - manifest = loader.load_manifest(args.project) - manifest_ctx = loader.ctx_gen.get_context(manifest.name) - run_tests = ( - args.enable_tests - and manifest.get("github.actions", "run_tests", ctx=manifest_ctx) != "off" - ) - rust_version = ( - manifest.get("github.actions", "rust_version", ctx=manifest_ctx) or "stable" - ) - - override_build_type = args.build_type or manifest.get( - "github.actions", "build_type", ctx=manifest_ctx - ) - if run_tests: - manifest_ctx.set("test", "on") - run_on = self.get_run_on(args) - - tests_arg = "--no-tests " - if run_tests: - tests_arg = "" - - # Some projects don't do anything "useful" as a leaf project, only - # as a dep for a leaf project. Check for those here; we don't want - # to waste the effort scheduling them on CI. - # We do this by looking at the builder type in the manifest file - # rather than creating a builder and checking its type because we - # don't know enough to create the full builder instance here. - builder_name = manifest.get("build", "builder", ctx=manifest_ctx) - if builder_name == "nop": - return None - - # We want to be sure that we're running things with python 3 - # but python versioning is honestly a bit of a frustrating mess. - # `python` may be version 2 or version 3 depending on the system. - # python3 may not be a thing at all! - # Assume an optimistic default - py3 = "python3" - - if build_opts.is_linux(): - artifacts = "linux" - if args.runs_on: - runs_on = args.runs_on - else: - runs_on = f"ubuntu-{args.ubuntu_version}" - if args.cpu_cores: - runs_on = f"{args.cpu_cores}-core-ubuntu-{args.ubuntu_version}" - elif build_opts.is_windows(): - artifacts = "windows" - if args.runs_on: - runs_on = args.runs_on - else: - runs_on = "windows-2022" - # The windows runners are python 3 by default; python2.exe - # is available if needed. - py3 = "python" - else: - artifacts = "mac" - if args.runs_on: - runs_on = args.runs_on - else: - runs_on = "macOS-latest" - - os.makedirs(args.output_dir, exist_ok=True) - - job_file_prefix = "getdeps_" - if args.job_file_prefix: - job_file_prefix = args.job_file_prefix - - output_file = os.path.join(args.output_dir, f"{job_file_prefix}{artifacts}.yml") - - if args.job_name_prefix: - job_name = args.job_name_prefix + artifacts.capitalize() - else: - job_name = artifacts - - with open(output_file, "w") as out: - # Deliberate line break here because the @ and the generated - # symbols are meaningful to our internal tooling when they - # appear in a single token - out.write("# This file was @") - out.write("generated by getdeps.py\n") - out.write( - f""" -name: {job_name} - -on:{run_on} - -permissions: - contents: read # to fetch code (actions/checkout) - -jobs: -""" - ) - - getdepscmd = f"{py3} build/fbcode_builder/getdeps.py" - - out.write(" build:\n") - out.write(" runs-on: %s\n" % runs_on) - out.write(" steps:\n") - - if build_opts.is_windows(): - # cmake relies on BOOST_ROOT but GH deliberately don't set it in order - # to avoid versioning issues: - # https://github.com/actions/virtual-environments/issues/319 - # Instead, set the version we think we need; this is effectively - # coupled with the boost manifest - # This is the unusual syntax for setting an env var for the rest of - # the steps in a workflow: - # https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/ - out.write(" - name: Export boost environment\n") - out.write( - ' run: "echo BOOST_ROOT=%BOOST_ROOT_1_83_0% >> %GITHUB_ENV%"\n' - ) - out.write(" shell: cmd\n") - - out.write(" - name: Fix Git config\n") - out.write(" run: >\n") - out.write(" git config --system core.longpaths true &&\n") - out.write(" git config --system core.autocrlf false &&\n") - # cxx crate needs symlinks enabled - out.write(" git config --system core.symlinks true\n") - # && is not supported on default windows powershell, so use cmd - out.write(" shell: cmd\n") - - out.write(" - uses: actions/checkout@v6\n") - - build_type_arg = "" - if override_build_type: - build_type_arg = f"--build-type {override_build_type} " - - if args.shared_libs: - build_type_arg += "--shared-libs " - - if build_opts.free_up_disk: - free_up_disk = "--free-up-disk " - if not build_opts.is_windows(): - out.write(" - name: Show disk space at start\n") - out.write(" run: df -h\n") - # remove the unused github supplied android dev tools - out.write(" - name: Free up disk space\n") - out.write(" run: sudo rm -rf /usr/local/lib/android\n") - out.write(" - name: Show disk space after freeing up\n") - out.write(" run: df -h\n") - else: - free_up_disk = "" - - allow_sys_arg = "" - if ( - build_opts.allow_system_packages - and build_opts.host_type.get_package_manager() - ): - sudo_arg = "sudo --preserve-env=http_proxy " - allow_sys_arg = " --allow-system-packages" - if build_opts.host_type.get_package_manager() == "deb": - out.write(" - name: Update system package info\n") - out.write(f" run: {sudo_arg}apt-get update\n") - - out.write(" - name: Install system deps\n") - if build_opts.is_darwin(): - # brew is installed as regular user - sudo_arg = "" - - system_deps_cmd = f"{sudo_arg}{getdepscmd}{allow_sys_arg} install-system-deps {tests_arg}--recursive {manifest.name}" - if build_opts.is_linux() or build_opts.is_freebsd(): - system_deps_cmd += f" && {sudo_arg}{getdepscmd}{allow_sys_arg} install-system-deps {tests_arg}--recursive patchelf" - out.write(f" run: {system_deps_cmd}\n") - - required_locales = manifest.get( - "github.actions", "required_locales", ctx=manifest_ctx - ) - if ( - build_opts.host_type.get_package_manager() == "deb" - and required_locales - ): - # ubuntu doesn't include this by default - out.write(" - name: Install locale-gen\n") - out.write(f" run: {sudo_arg}apt-get install locales\n") - for loc in required_locales.split(): - out.write(f" - name: Ensure {loc} locale present\n") - out.write(f" run: {sudo_arg}locale-gen {loc}\n") - - out.write(" - id: paths\n") - out.write(" name: Query paths\n") - if build_opts.is_windows(): - out.write( - f" run: {getdepscmd}{allow_sys_arg} query-paths {tests_arg}--recursive --src-dir=. {manifest.name} >> $env:GITHUB_OUTPUT\n" - ) - out.write(" shell: pwsh\n") - else: - out.write( - f' run: {getdepscmd}{allow_sys_arg} query-paths {tests_arg}--recursive --src-dir=. {manifest.name} >> "$GITHUB_OUTPUT"\n' - ) - - projects = loader.manifests_in_dependency_order() - - main_repo_url = manifest.get_repo_url(manifest_ctx) - has_same_repo_dep = False - - # Add the rust dep which doesn't have a manifest - for m in projects: - if m == manifest: - continue - mbuilder_name = m.get("build", "builder", ctx=manifest_ctx) - if ( - m.name == "rust" - or builder_name == "cargo" - or mbuilder_name == "cargo" - ): - out.write(f" - name: Install Rust {rust_version.capitalize()}\n") - out.write(f" uses: dtolnay/rust-toolchain@{rust_version}\n") - break - - # Normal deps that have manifests - for m in projects: - if m == manifest or m.name == "rust": - continue - ctx = loader.ctx_gen.get_context(m.name) - if m.get_repo_url(ctx) != main_repo_url: - out.write(" - name: Fetch %s\n" % m.name) - out.write( - f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE }}}}\n" - ) - out.write( - f" run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n" - ) - - for m in projects: - if m == manifest or m.name == "rust": - continue - src_dir_arg = "" - ctx = loader.ctx_gen.get_context(m.name) - if main_repo_url and m.get_repo_url(ctx) == main_repo_url: - # Its in the same repo, so src-dir is also . - src_dir_arg = "--src-dir=. " - has_same_repo_dep = True - - if args.use_build_cache and not src_dir_arg: - out.write(f" - name: Restore {m.name} from cache\n") - out.write(f" id: restore_{m.name}\n") - # only need to restore if would build it - out.write( - f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE }}}}\n" - ) - out.write(" uses: actions/cache/restore@v4\n") - out.write(" with:\n") - out.write( - f" path: ${{{{ steps.paths.outputs.{m.name}_INSTALL }}}}\n" - ) - out.write( - f" key: ${{{{ steps.paths.outputs.{m.name}_CACHE_KEY }}}}-install\n" - ) - - out.write(" - name: Build %s\n" % m.name) - if not src_dir_arg: - if args.use_build_cache: - out.write( - f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE && ! steps.restore_{m.name}.outputs.cache-hit }}}}\n" - ) - else: - out.write( - f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE }}}}\n" - ) - out.write( - f" run: {getdepscmd}{allow_sys_arg} build {build_type_arg}{src_dir_arg}{free_up_disk}--no-tests {m.name}\n" - ) - - if args.use_build_cache and not src_dir_arg: - out.write(f" - name: Save {m.name} to cache\n") - out.write(" uses: actions/cache/save@v4\n") - out.write( - f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE && ! steps.restore_{m.name}.outputs.cache-hit }}}}\n" - ) - out.write(" with:\n") - out.write( - f" path: ${{{{ steps.paths.outputs.{m.name}_INSTALL }}}}\n" - ) - out.write( - f" key: ${{{{ steps.paths.outputs.{m.name}_CACHE_KEY }}}}-install\n" - ) - - out.write(" - name: Build %s\n" % manifest.name) - - project_prefix = "" - if not build_opts.is_windows(): - prefix = loader.get_project_install_prefix(manifest) or "/usr/local" - project_prefix = " --project-install-prefix %s:%s" % ( - manifest.name, - prefix, - ) - - # If we have dep from same repo, we already built it and don't want to rebuild it again - no_deps_arg = "" - if has_same_repo_dep: - no_deps_arg = "--no-deps " - - out.write( - f" run: {getdepscmd}{allow_sys_arg} build {build_type_arg}{tests_arg}{no_deps_arg}--src-dir=. {manifest.name}{project_prefix}\n" - ) - - out.write(" - name: Copy artifacts\n") - if build_opts.is_linux(): - # Strip debug info from the binaries, but only on linux. - # While the `strip` utility is also available on macOS, - # attempting to strip there results in an error. - # The `strip` utility is not available on Windows. - strip = " --strip" - else: - strip = "" - - out.write( - f" run: {getdepscmd}{allow_sys_arg} fixup-dyn-deps{strip} " - f"--src-dir=. {manifest.name} _artifacts/{artifacts}{project_prefix} " - f"--final-install-prefix /usr/local\n" - ) - - out.write(" - uses: actions/upload-artifact@v6\n") - out.write(" with:\n") - out.write(" name: %s\n" % manifest.name) - out.write(" path: _artifacts\n") - - if run_tests: - num_jobs_arg = "" - if args.num_jobs: - num_jobs_arg = f"--num-jobs {args.num_jobs} " - - out.write(" - name: Test %s\n" % manifest.name) - out.write( - f" run: {getdepscmd}{allow_sys_arg} test {build_type_arg}{num_jobs_arg}--src-dir=. {manifest.name}{project_prefix}\n" - ) - if build_opts.free_up_disk and not build_opts.is_windows(): - out.write(" - name: Show disk space at end\n") - out.write(" if: always()\n") - out.write(" run: df -h\n") - - def setup_project_cmd_parser(self, parser): - parser.add_argument( - "--disallow-system-packages", - help="Disallow satisfying third party deps from installed system packages", - action="store_true", - default=False, - ) - parser.add_argument( - "--output-dir", help="The directory that will contain the yml files" - ) - parser.add_argument( - "--run-on-all-branches", - action="store_true", - help="Allow CI to fire on all branches - Handy for testing", - ) - parser.add_argument( - "--ubuntu-version", default="22.04", help="Version of Ubuntu to use" - ) - parser.add_argument( - "--cpu-cores", - help="Number of CPU cores to use (applicable for Linux OS)", - ) - parser.add_argument( - "--runs-on", - help="Allow specifying explicit runs-on: for github actions", - ) - parser.add_argument( - "--cron", - help="Specify that the job runs on a cron schedule instead of on pushes. Pass never to disable the action.", - ) - parser.add_argument( - "--main-branch", - default="main", - help="Main branch to trigger GitHub Action on", - ) - parser.add_argument( - "--os-type", - help="Filter to just this OS type to run", - choices=["linux", "darwin", "windows"], - action="append", - dest="os_types", - default=[], - ) - parser.add_argument( - "--job-file-prefix", - type=str, - help="add a prefix to all job file names", - default=None, - ) - parser.add_argument( - "--job-name-prefix", - type=str, - help="add a prefix to all job names", - default=None, - ) - parser.add_argument( - "--free-up-disk", - help="Remove unused tools and clean up intermediate files if possible to maximise space for the build", - action="store_true", - default=False, - ) - parser.add_argument( - "--build-type", - help="Set the build type explicitly. Cmake and cargo builders act on them. Only Debug and RelWithDebInfo widely supported.", - choices=["Debug", "Release", "RelWithDebInfo", "MinSizeRel"], - action="store", - default=None, - ) - parser.add_argument( - "--no-build-cache", - action="store_false", - default=True, - dest="use_build_cache", - help="Do not attempt to use the build cache.", - ) - - -def get_arg_var_name(args): - for arg in args: - if arg.startswith("--"): - return arg[2:].replace("-", "_") - - raise Exception("unable to determine argument variable name from %r" % (args,)) - - -def parse_args(): - # We want to allow common arguments to be specified either before or after - # the subcommand name. In order to do this we add them to the main parser - # and to subcommand parsers. In order for this to work, we need to tell - # argparse that the default value is SUPPRESS, so that the default values - # from the subparser arguments won't override values set by the user from - # the main parser. We maintain our own list of desired defaults in the - # common_defaults dictionary, and manually set those if the argument wasn't - # present at all. - common_args = argparse.ArgumentParser(add_help=False) - common_defaults = {} - - def add_common_arg(*args, **kwargs): - var_name = get_arg_var_name(args) - default_value = kwargs.pop("default", None) - common_defaults[var_name] = default_value - kwargs["default"] = argparse.SUPPRESS - common_args.add_argument(*args, **kwargs) - - add_common_arg("--scratch-path", help="Where to maintain checkouts and build dirs") - add_common_arg( - "--vcvars-path", default=None, help="Path to the vcvarsall.bat on Windows." - ) - add_common_arg( - "--install-prefix", - help=( - "Where the final build products will be installed " - "(default is [scratch-path]/installed)" - ), - ) - add_common_arg( - "--num-jobs", - type=int, - help=( - "Number of concurrent jobs to use while building. " - "(default=number of cpu cores)" - ), - ) - add_common_arg( - "--use-shipit", - help="use the real ShipIt instead of the simple shipit transformer", - action="store_true", - default=False, - ) - add_common_arg( - "--facebook-internal", - help="Setup the build context as an FB internal build", - action="store_true", - default=None, - ) - add_common_arg( - "--no-facebook-internal", - help="Perform a non-FB internal build, even when in an fbsource repository", - action="store_false", - dest="facebook_internal", - ) - add_common_arg( - "--shared-libs", - help="Build shared libraries if possible", - action="store_true", - default=False, - ) - add_common_arg( - "--extra-cmake-defines", - help=( - "Input json map that contains extra cmake defines to be used " - "when compiling the current project and all its deps. " - 'e.g: \'{"CMAKE_CXX_FLAGS": "--bla"}\'' - ), - ) - add_common_arg( - "--allow-system-packages", - help="Allow satisfying third party deps from installed system packages", - action="store_true", - default=False, - ) - add_common_arg( - "-v", - "--verbose", - help="Print more output", - action="store_true", - default=False, - ) - add_common_arg( - "-su", - "--skip-upload", - help="skip upload steps", - action="store_true", - default=False, - ) - add_common_arg( - "--lfs-path", - help="Provide a parent directory for lfs when fbsource is unavailable", - default=None, - ) - add_common_arg( - "--build-skip-lfs-download", - action="store_true", - default=False, - help=( - "Download from the URL, rather than LFS. This is useful " - "in cases where the upstream project has uploaded a new " - "version of the archive with a different hash" - ), - ) - add_common_arg( - "--schedule-type", - nargs="?", - help="Indicates how the build was activated", - ) - - ap = argparse.ArgumentParser( - description="Get and build dependencies and projects", parents=[common_args] - ) - sub = ap.add_subparsers( - # metavar suppresses the long and ugly default list of subcommands on a - # single line. We still render the nicer list below where we would - # have shown the nasty one. - metavar="", - title="Available commands", - help="", - ) - - add_subcommands(sub, common_args) - - args = ap.parse_args() - for var_name, default_value in common_defaults.items(): - if not hasattr(args, var_name): - setattr(args, var_name, default_value) - - return ap, args - - -def main(): - ap, args = parse_args() - if getattr(args, "func", None) is None: - ap.print_help() - return 0 - try: - return args.func(args) - except UsageError as exc: - ap.error(str(exc)) - return 1 - except TransientFailure as exc: - print("TransientFailure: %s" % str(exc)) - # This return code is treated as a retryable transient infrastructure - # error by Facebook's internal CI, rather than eg: a build or code - # related error that needs to be fixed before progress can be made. - return 128 - except subprocess.CalledProcessError as exc: - print("%s" % str(exc), file=sys.stderr) - print("!! Failed", file=sys.stderr) - return 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py deleted file mode 100644 index a5e18fade..000000000 --- a/build/fbcode_builder/getdeps/builder.py +++ /dev/null @@ -1,1554 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-unsafe - -import glob -import json -import os -import pathlib -import shutil -import stat -import subprocess -import sys -import typing -from shlex import quote as shellquote -from typing import Optional - -from .copytree import rmtree_more, simple_copytree -from .dyndeps import create_dyn_dep_munger -from .envfuncs import add_path_entry, Env, path_search -from .fetcher import copy_if_different, is_public_commit -from .runcmd import run_cmd - -if typing.TYPE_CHECKING: - from .buildopts import BuildOptions - - -class BuilderBase(object): - def __init__( - self, - loader, - dep_manifests, # manifests of dependencies - build_opts: "BuildOptions", - ctx, - manifest, - src_dir, - build_dir, - inst_dir, - env=None, - final_install_prefix=None, - ) -> None: - self.env = Env() - if env: - self.env.update(env) - - subdir = manifest.get("build", "subdir", ctx=ctx) - if subdir: - src_dir = os.path.join(src_dir, subdir) - - self.patchfile = manifest.get("build", "patchfile", ctx=ctx) - self.patchfile_opts = manifest.get("build", "patchfile_opts", ctx=ctx) or "" - self.ctx = ctx - self.src_dir = src_dir - self.build_dir = build_dir or src_dir - self.inst_dir = inst_dir - self.build_opts = build_opts - self.manifest = manifest - self.final_install_prefix = final_install_prefix - self.loader = loader - self.dep_manifests = dep_manifests - self.install_dirs = [loader.get_project_install_dir(m) for m in dep_manifests] - - def _get_cmd_prefix(self): - if self.build_opts.is_windows(): - vcvarsall = self.build_opts.get_vcvars_path() - if vcvarsall is not None: - # Since it sets rather a large number of variables we mildly abuse - # the cmd quoting rules to assemble a command that calls the script - # to prep the environment and then triggers the actual command that - # we wanted to run. - - # Due to changes in vscrsall.bat, it now reports an ERRORLEVEL of 1 - # even when succeeding. This occurs when an extension is not present. - # To continue, we must ignore the ERRORLEVEL returned. We do this by - # wrapping the call in a batch file that always succeeds. - wrapper = os.path.join(self.build_dir, "succeed.bat") - with open(wrapper, "w") as f: - f.write("@echo off\n") - f.write(f'call "{vcvarsall}" amd64\n') - f.write("set ERRORLEVEL=0\n") - f.write("exit /b 0\n") - return [wrapper, "&&"] - return [] - - def _check_cmd(self, cmd, **kwargs) -> None: - """Run the command and abort on failure""" - rc = self._run_cmd(cmd, **kwargs) - if rc != 0: - raise RuntimeError(f"Failure exit code {rc} for command {cmd}") - - def _run_cmd( - self, - cmd, - cwd=None, - env=None, - use_cmd_prefix: bool = True, - allow_fail: bool = False, - ) -> int: - if env: - e = self.env.copy() - e.update(env) - env = e - else: - env = self.env - - if use_cmd_prefix: - cmd_prefix = self._get_cmd_prefix() - if cmd_prefix: - cmd = cmd_prefix + cmd - - log_file = os.path.join(self.build_dir, "getdeps_build.log") - return run_cmd( - cmd=cmd, - env=env, - cwd=cwd or self.build_dir, - log_file=log_file, - allow_fail=allow_fail, - ) - - def _reconfigure(self, reconfigure: bool) -> bool: - if self.build_dir is not None: - if not os.path.isdir(self.build_dir): - os.makedirs(self.build_dir) - reconfigure = True - return reconfigure - - def _apply_patchfile(self) -> None: - if self.patchfile is None: - return - patched_sentinel_file = pathlib.Path(self.src_dir + "/.getdeps_patched") - if patched_sentinel_file.exists(): - return - old_wd = os.getcwd() - os.chdir(self.src_dir) - print(f"Patching {self.manifest.name} with {self.patchfile} in {self.src_dir}") - patchfile = os.path.join( - self.build_opts.fbcode_builder_dir, "patches", self.patchfile - ) - patchcmd = ["git", "apply", "--ignore-space-change"] - if self.patchfile_opts: - patchcmd.append(self.patchfile_opts) - try: - subprocess.check_call(patchcmd + [patchfile]) - except subprocess.CalledProcessError: - raise ValueError(f"Failed to apply patch to {self.manifest.name}") - os.chdir(old_wd) - patched_sentinel_file.touch() - - def prepare(self, reconfigure: bool) -> None: - print("Preparing %s..." % self.manifest.name) - reconfigure = self._reconfigure(reconfigure) - self._apply_patchfile() - self._prepare(reconfigure=reconfigure) - - def debug(self, reconfigure: bool) -> None: - reconfigure = self._reconfigure(reconfigure) - self._apply_patchfile() - self._prepare(reconfigure=reconfigure) - env = self._compute_env() - print("Starting a shell in %s, ^D to exit..." % self.build_dir) - # TODO: print the command to run the build - shell = ["powershell.exe"] if sys.platform == "win32" else ["/bin/sh", "-i"] - self._run_cmd(shell, cwd=self.build_dir, env=env) - - def printenv(self, reconfigure: bool) -> None: - """print the environment in a shell sourcable format""" - reconfigure = self._reconfigure(reconfigure) - self._apply_patchfile() - self._prepare(reconfigure=reconfigure) - env = self._compute_env(env=Env(src={})) - prefix = "export " - sep = ":" - expand = "$" - expandpost = "" - if self.build_opts.is_windows(): - prefix = "SET " - sep = ";" - expand = "%" - expandpost = "%" - for k, v in sorted(env.items()): - existing = os.environ.get(k, None) - if k.endswith("PATH") and existing: - v = shellquote(v) + sep + f"{expand}{k}{expandpost}" - else: - v = shellquote(v) - print("%s%s=%s" % (prefix, k, v)) - - def build(self, reconfigure: bool) -> None: - print("Building %s..." % self.manifest.name) - reconfigure = self._reconfigure(reconfigure) - self._apply_patchfile() - self._prepare(reconfigure=reconfigure) - self._build(reconfigure=reconfigure) - - if self.build_opts.free_up_disk: - # don't clean --src-dir=. case as user may want to build again or run tests on the build - if self.src_dir.startswith(self.build_opts.scratch_dir) and os.path.isdir( - self.build_dir - ): - if os.path.islink(self.build_dir): - os.remove(self.build_dir) - else: - rmtree_more(self.build_dir) - elif self.build_opts.is_windows(): - # On Windows, emit a wrapper script that can be used to run build artifacts - # directly from the build directory, without installing them. On Windows $PATH - # needs to be updated to include all of the directories containing the runtime - # library dependencies in order to run the binaries. - script_path = self.get_dev_run_script_path() - dep_munger = create_dyn_dep_munger( - self.build_opts, self._compute_env(), self.install_dirs - ) - dep_dirs = self.get_dev_run_extra_path_dirs(dep_munger) - # pyre-fixme[16]: Optional type has no attribute `emit_dev_run_script`. - dep_munger.emit_dev_run_script(script_path, dep_dirs) - - @property - def num_jobs(self) -> int: - # This is a hack, but we don't have a "defaults manifest" that we can - # customize per platform. - # TODO: Introduce some sort of defaults config that can select by - # platform, just like manifest contexts. - if sys.platform.startswith("freebsd"): - # clang on FreeBSD is quite memory-efficient. - default_job_weight = 512 - else: - # 1.5 GiB is a lot to assume, but it's typical of Facebook-style C++. - # Some manifests are even heavier and should override. - default_job_weight = 1536 - return self.build_opts.get_num_jobs( - int( - self.manifest.get( - "build", "job_weight_mib", default_job_weight, ctx=self.ctx - ) - ) - ) - - def run_tests( - self, - schedule_type, - owner, - test_filter, - test_exclude, - retry, - no_testpilot, - timeout=None, - ) -> None: - """Execute any tests that we know how to run. If they fail, - raise an exception.""" - pass - - def _prepare(self, reconfigure) -> None: - """Prepare the build. Useful when need to generate config, - but builder is not the primary build system. - e.g. cargo when called from cmake""" - pass - - def _build(self, reconfigure) -> None: - """Perform the build. - reconfigure will be set to true if the fetcher determined - that the sources have changed in such a way that the build - system needs to regenerate its rules.""" - pass - - def _compute_env(self, env=None) -> Env: - if env is None: - env = self.env - # CMAKE_PREFIX_PATH is only respected when passed through the - # environment, so we construct an appropriate path to pass down - return self.build_opts.compute_env_for_install_dirs( - self.loader, - self.dep_manifests, - self.ctx, - env=env, - manifest=self.manifest, - ) - - def get_dev_run_script_path(self): - assert self.build_opts.is_windows() - return os.path.join(self.build_dir, "run.ps1") - - def get_dev_run_extra_path_dirs(self, dep_munger=None): - assert self.build_opts.is_windows() - if dep_munger is None: - dep_munger = create_dyn_dep_munger( - self.build_opts, self._compute_env(), self.install_dirs - ) - return dep_munger.compute_dependency_paths(self.build_dir) - - -class MakeBuilder(BuilderBase): - def __init__( - self, - loader, - dep_manifests, - build_opts, - ctx, - manifest, - src_dir, - build_dir, - inst_dir, - build_args, - install_args, - test_args, - ) -> None: - super(MakeBuilder, self).__init__( - loader, - dep_manifests, - build_opts, - ctx, - manifest, - src_dir, - build_dir, - inst_dir, - ) - self.build_args = build_args or [] - self.install_args = install_args or [] - self.test_args = test_args - - @property - def _make_binary(self): - return self.manifest.get("build", "make_binary", "make", ctx=self.ctx) - - def _get_prefix(self): - return ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir] - - def _build(self, reconfigure) -> None: - - env = self._compute_env() - - # Need to ensure that PREFIX is set prior to install because - # libbpf uses it when generating its pkg-config file. - # The lowercase prefix is used by some projects. - cmd = ( - [self._make_binary, "-j%s" % self.num_jobs] - + self.build_args - + self._get_prefix() - ) - self._check_cmd(cmd, env=env) - - install_cmd = [self._make_binary] + self.install_args + self._get_prefix() - self._check_cmd(install_cmd, env=env) - - # bz2's Makefile doesn't install its .so properly - if self.manifest and self.manifest.name == "bz2": - libdir = os.path.join(self.inst_dir, "lib") - srcpattern = os.path.join(self.src_dir, "lib*.so.*") - print(f"copying to {libdir} from {srcpattern}") - for file in glob.glob(srcpattern): - shutil.copy(file, libdir) - - def run_tests( - self, - schedule_type, - owner, - test_filter, - test_exclude, - retry, - no_testpilot, - timeout=None, - ) -> None: - if not self.test_args: - return - - env = self._compute_env() - if test_filter: - env["GETDEPS_TEST_FILTER"] = test_filter - else: - env["GETDEPS_TEST_FILTER"] = "" - - if retry: - env["GETDEPS_TEST_RETRY"] = retry - else: - env["GETDEPS_TEST_RETRY"] = 0 - - if timeout is not None: - env["GETDEPS_TEST_TIMEOUT"] = str(timeout) - - cmd = ( - [self._make_binary, "-j%s" % self.num_jobs] - + self.test_args - + self._get_prefix() - ) - self._check_cmd(cmd, allow_fail=False, env=env) - - -class CMakeBootStrapBuilder(MakeBuilder): - def _build(self, reconfigure) -> None: - self._check_cmd( - [ - "./bootstrap", - "--prefix=" + self.inst_dir, - f"--parallel={self.num_jobs}", - ] - ) - super(CMakeBootStrapBuilder, self)._build(reconfigure) - - -class AutoconfBuilder(BuilderBase): - def __init__( - self, - loader, - dep_manifests, - build_opts, - ctx, - manifest, - src_dir, - build_dir, - inst_dir, - args, - conf_env_args, - ) -> None: - super(AutoconfBuilder, self).__init__( - loader, - dep_manifests, - build_opts, - ctx, - manifest, - src_dir, - build_dir, - inst_dir, - ) - self.args = args or [] - self.conf_env_args = conf_env_args or {} - - @property - def _make_binary(self): - return self.manifest.get("build", "make_binary", "make", ctx=self.ctx) - - def _build(self, reconfigure) -> None: - configure_path = os.path.join(self.src_dir, "configure") - autogen_path = os.path.join(self.src_dir, "autogen.sh") - - env = self._compute_env() - - # Some configure scripts need additional env values passed derived from cmds - for k, cmd_args in self.conf_env_args.items(): - out = ( - subprocess.check_output(cmd_args, env=dict(env.items())) - .decode("utf-8") - .strip() - ) - if out: - env.set(k, out) - - if not os.path.exists(configure_path): - print("%s doesn't exist, so reconfiguring" % configure_path) - # This libtoolize call is a bit gross; the issue is that - # `autoreconf` as invoked by libsodium's `autogen.sh` doesn't - # seem to realize that it should invoke libtoolize and then - # error out when the configure script references a libtool - # related symbol. - self._check_cmd(["libtoolize"], cwd=self.src_dir, env=env) - - # We generally prefer to call the `autogen.sh` script provided - # by the project on the basis that it may know more than plain - # autoreconf does. - if os.path.exists(autogen_path): - self._check_cmd(["bash", autogen_path], cwd=self.src_dir, env=env) - else: - self._check_cmd(["autoreconf", "-ivf"], cwd=self.src_dir, env=env) - configure_cmd = [configure_path, "--prefix=" + self.inst_dir] + self.args - self._check_cmd(configure_cmd, env=env) - only_install = self.manifest.get("build", "only_install", ctx=self.ctx) - if not only_install or only_install.lower() == "false": - self._check_cmd([self._make_binary, "-j%s" % self.num_jobs], env=env) - self._check_cmd([self._make_binary, "install"], env=env) - - -class Iproute2Builder(BuilderBase): - # ./configure --prefix does not work for iproute2. - # Thus, explicitly copy sources from src_dir to build_dir, build, - # and then install to inst_dir using DESTDIR - # lastly, also copy include from build_dir to inst_dir - def __init__( - self, - loader, - dep_manifests, - build_opts, - ctx, - manifest, - src_dir, - build_dir, - inst_dir, - ) -> None: - super(Iproute2Builder, self).__init__( - loader, - dep_manifests, - build_opts, - ctx, - manifest, - src_dir, - build_dir, - inst_dir, - ) - - def _build(self, reconfigure) -> None: - configure_path = os.path.join(self.src_dir, "configure") - env = self.env.copy() - self._check_cmd([configure_path], env=env) - shutil.rmtree(self.build_dir) - shutil.copytree(self.src_dir, self.build_dir) - self._check_cmd(["make", "-j%s" % self.num_jobs], env=env) - install_cmd = ["make", "install", "DESTDIR=" + self.inst_dir] - - for d in ["include", "lib"]: - if not os.path.isdir(os.path.join(self.inst_dir, d)): - shutil.copytree( - os.path.join(self.build_dir, d), os.path.join(self.inst_dir, d) - ) - - self._check_cmd(install_cmd, env=env) - - -class MesonBuilder(BuilderBase): - # MesonBuilder assumes that meson build tool has already been installed on - # the machine. - def __init__( - self, - loader, - dep_manifests, - build_opts, - ctx, - manifest, - src_dir, - build_dir, - inst_dir, - ) -> None: - super(MesonBuilder, self).__init__( - loader, - dep_manifests, - build_opts, - ctx, - manifest, - src_dir, - build_dir, - inst_dir, - ) - - def _build(self, reconfigure) -> None: - env = self._compute_env() - meson = path_search(env, "meson") - if meson is None: - raise Exception("Failed to find Meson") - - setup_args = self.manifest.get_section_as_args("meson.setup_args", self.ctx) - - # Meson builds typically require setup, compile, and install steps. - # During this setup step we ensure that the static library is built and - # the prefix is empty. - self._check_cmd( - [ - meson, - "setup", - ] - + setup_args - + [ - self.build_dir, - self.src_dir, - ] - ) - - # Compile step needs to satisfy the build directory that was previously - # prepared during setup. - self._check_cmd([meson, "compile", "-C", self.build_dir]) - - # Install step - self._check_cmd( - [meson, "install", "-C", self.build_dir, "--destdir", self.inst_dir] - ) - - -class CMakeBuilder(BuilderBase): - MANUAL_BUILD_SCRIPT = """\ -#!{sys.executable} - - -import argparse -import subprocess -import sys - -CMAKE = {cmake!r} -CTEST = {ctest!r} -SRC_DIR = {src_dir!r} -BUILD_DIR = {build_dir!r} -INSTALL_DIR = {install_dir!r} -CMD_PREFIX = {cmd_prefix!r} -CMAKE_ENV = {env_str} -CMAKE_DEFINE_ARGS = {define_args_str} - - -def get_jobs_argument(num_jobs_arg: int) -> str: - if num_jobs_arg > 0: - return "-j" + str(num_jobs_arg) - - import multiprocessing - num_jobs = multiprocessing.cpu_count() // 2 - return "-j" + str(num_jobs) - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument( - "cmake_args", - nargs=argparse.REMAINDER, - help='Any extra arguments after an "--" argument will be passed ' - "directly to CMake." - ) - ap.add_argument( - "--mode", - choices=["configure", "build", "install", "test"], - default="configure", - help="The mode to run: configure, build, or install. " - "Defaults to configure", - ) - ap.add_argument( - "--build", - action="store_const", - const="build", - dest="mode", - help="An alias for --mode=build", - ) - ap.add_argument( - "-j", - "--num-jobs", - action="store", - type=int, - default=0, - help="Run the build or tests with the specified number of parallel jobs", - ) - ap.add_argument( - "--install", - action="store_const", - const="install", - dest="mode", - help="An alias for --mode=install", - ) - ap.add_argument( - "--test", - action="store_const", - const="test", - dest="mode", - help="An alias for --mode=test", - ) - args = ap.parse_args() - - # Strip off a leading "--" from the additional CMake arguments - if args.cmake_args and args.cmake_args[0] == "--": - args.cmake_args = args.cmake_args[1:] - - env = CMAKE_ENV - - if args.mode == "configure": - full_cmd = CMD_PREFIX + [CMAKE, SRC_DIR] + CMAKE_DEFINE_ARGS + args.cmake_args - elif args.mode in ("build", "install"): - target = "all" if args.mode == "build" else "install" - full_cmd = CMD_PREFIX + [ - CMAKE, - "--build", - BUILD_DIR, - "--target", - target, - "--config", - "{build_type}", - get_jobs_argument(args.num_jobs), - ] + args.cmake_args - elif args.mode == "test": - full_cmd = CMD_PREFIX + [ - {dev_run_script}CTEST, - "--output-on-failure", - get_jobs_argument(args.num_jobs), - ] + args.cmake_args - else: - ap.error("unknown invocation mode: %s" % (args.mode,)) - - cmd_str = " ".join(full_cmd) - print("Running: %r" % (cmd_str,)) - proc = subprocess.run(full_cmd, env=env, cwd=BUILD_DIR) - sys.exit(proc.returncode) - - -if __name__ == "__main__": - main() -""" - - def __init__( - self, - loader, - dep_manifests, - build_opts, - ctx, - manifest, - src_dir, - build_dir, - inst_dir, - defines, - final_install_prefix=None, - extra_cmake_defines=None, - cmake_targets=None, - ) -> None: - super(CMakeBuilder, self).__init__( - loader, - dep_manifests, - build_opts, - ctx, - manifest, - src_dir, - build_dir, - inst_dir, - final_install_prefix=final_install_prefix, - ) - self.defines = defines or {} - if extra_cmake_defines: - self.defines.update(extra_cmake_defines) - self.cmake_targets = cmake_targets or ["install"] - - if build_opts.is_windows(): - try: - from .facebook.vcvarsall import extra_vc_cmake_defines - except ImportError: - pass - else: - self.defines.update(extra_vc_cmake_defines) - - self.loader = loader - if build_opts.shared_libs: - self.defines["BUILD_SHARED_LIBS"] = "ON" - self.defines["BOOST_LINK_STATIC"] = "OFF" - - def _invalidate_cache(self) -> None: - for name in [ - "CMakeCache.txt", - "CMakeFiles/CMakeError.log", - "CMakeFiles/CMakeOutput.log", - ]: - name = os.path.join(self.build_dir, name) - if os.path.isdir(name): - shutil.rmtree(name) - elif os.path.exists(name): - os.unlink(name) - - def _needs_reconfigure(self) -> bool: - for name in ["CMakeCache.txt", "build.ninja"]: - name = os.path.join(self.build_dir, name) - if not os.path.exists(name): - return True - return False - - def _write_build_script(self, **kwargs) -> None: - env_lines = [" {!r}: {!r},".format(k, v) for k, v in kwargs["env"].items()] - kwargs["env_str"] = "\n".join(["{"] + env_lines + ["}"]) - - if self.build_opts.is_windows(): - kwargs["dev_run_script"] = '"powershell.exe", {!r}, '.format( - self.get_dev_run_script_path() - ) - else: - kwargs["dev_run_script"] = "" - - define_arg_lines = ["["] - for arg in kwargs["define_args"]: - # Replace the CMAKE_INSTALL_PREFIX argument to use the INSTALL_DIR - # variable that we define in the MANUAL_BUILD_SCRIPT code. - if arg.startswith("-DCMAKE_INSTALL_PREFIX="): - value = " {!r}.format(INSTALL_DIR),".format( - "-DCMAKE_INSTALL_PREFIX={}" - ) - else: - value = " {!r},".format(arg) - define_arg_lines.append(value) - define_arg_lines.append("]") - kwargs["define_args_str"] = "\n".join(define_arg_lines) - - # In order to make it easier for developers to manually run builds for - # CMake-based projects, write out some build scripts that can be used to invoke - # CMake manually. - build_script_path = os.path.join(self.build_dir, "run_cmake.py") - script_contents = self.MANUAL_BUILD_SCRIPT.format(**kwargs) - with open(build_script_path, "wb") as f: - f.write(script_contents.encode()) - os.chmod(build_script_path, 0o755) - - def _compute_cmake_define_args(self, env): - defines = { - "CMAKE_INSTALL_PREFIX": self.final_install_prefix or self.inst_dir, - "BUILD_SHARED_LIBS": "OFF", - # Some of the deps (rsocket) default to UBSAN enabled if left - # unspecified. Some of the deps fail to compile in release mode - # due to warning->error promotion. RelWithDebInfo is the happy - # medium. - "CMAKE_BUILD_TYPE": self.build_opts.build_type, - } - - if "SANDCASTLE" not in os.environ: - # We sometimes see intermittent ccache related breakages on some - # of the FB internal CI hosts, so we prefer to disable ccache - # when running in that environment. - ccache = path_search(env, "ccache") - if ccache: - defines["CMAKE_CXX_COMPILER_LAUNCHER"] = ccache - else: - # rocksdb does its own probing for ccache. - # Ensure that it is disabled on sandcastle - env["CCACHE_DISABLE"] = "1" - # Some sandcastle hosts have broken ccache related dirs, and - # even though we've asked for it to be disabled ccache is - # still invoked by rocksdb's cmake. - # Redirect its config directory to somewhere that is guaranteed - # fresh to us, and that won't have any ccache data inside. - env["CCACHE_DIR"] = f"{self.build_opts.scratch_dir}/ccache" - - if "GITHUB_ACTIONS" in os.environ and self.build_opts.is_windows(): - # GitHub actions: the host has both gcc and msvc installed, and - # the default behavior of cmake is to prefer gcc. - # Instruct cmake that we want it to use cl.exe; this is important - # because Boost prefers cl.exe and the mismatch results in cmake - # with gcc not being able to find boost built with cl.exe. - defines["CMAKE_C_COMPILER"] = "cl.exe" - defines["CMAKE_CXX_COMPILER"] = "cl.exe" - - if self.build_opts.is_darwin(): - # Try to persuade cmake to set the rpath to match the lib - # dirs of the dependencies. This isn't automatic, and to - # make things more interesting, cmake uses `;` as the path - # separator, so translate the runtime path to something - # that cmake will parse - defines["CMAKE_INSTALL_RPATH"] = ";".join( - env.get("DYLD_LIBRARY_PATH", "").split(":") - ) - # Tell cmake that we want to set the rpath in the tree - # at build time. Without this the rpath is only set - # at the moment that the binaries are installed. That - # default is problematic for example when using the - # gtest integration in cmake which runs the built test - # executables during the build to discover the set of - # tests. - defines["CMAKE_BUILD_WITH_INSTALL_RPATH"] = "ON" - - defines.update(self.defines) - define_args = ["-D%s=%s" % (k, v) for (k, v) in defines.items()] - - # if self.build_opts.is_windows(): - # define_args += ["-G", "Visual Studio 15 2017 Win64"] - define_args += ["-G", "Ninja"] - - return define_args - - def _run_include_rewriter(self): - """Run include path rewriting on source files before building.""" - from .include_rewriter import rewrite_includes_from_manifest - - print(f"Rewriting include paths for {self.manifest.name}...") - try: - modified_count = rewrite_includes_from_manifest( - self.manifest, self.ctx, self.src_dir, verbose=True - ) - if modified_count > 0: - print(f"Successfully modified {modified_count} files") - else: - print("No files needed modification") - except Exception as e: - print(f"Warning: Include path rewriting failed: {e}") - # Don't fail the build for include rewriting issues - - def _build(self, reconfigure: bool) -> None: - # Check if include rewriting is enabled - rewrite_includes = self.manifest.get( - "build", "rewrite_includes", "false", ctx=self.ctx - ) - if rewrite_includes.lower() == "true": - self._run_include_rewriter() - - reconfigure = reconfigure or self._needs_reconfigure() - - env = self._compute_env() - if not self.build_opts.is_windows() and self.final_install_prefix: - env["DESTDIR"] = self.inst_dir - - # Resolve the cmake that we installed - cmake = path_search(env, "cmake") - if cmake is None: - raise Exception("Failed to find CMake") - - if self.build_opts.is_windows(): - checkdir = self.src_dir - if os.path.exists(checkdir): - children = os.listdir(checkdir) - print(f"Building from source {checkdir} contents: {children}") - else: - print(f"Source {checkdir} not found") - - if reconfigure: - define_args = self._compute_cmake_define_args(env) - self._write_build_script( - cmd_prefix=self._get_cmd_prefix(), - cmake=cmake, - ctest=path_search(env, "ctest"), - env=env, - define_args=define_args, - src_dir=self.src_dir, - build_dir=self.build_dir, - install_dir=self.inst_dir, - sys=sys, - build_type=self.build_opts.build_type, - ) - - self._invalidate_cache() - self._check_cmd([cmake, self.src_dir] + define_args, env=env) - - self._check_cmd( - [cmake, "--build", self.build_dir, "--target"] - + self.cmake_targets - + [ - "--config", - self.build_opts.build_type, - "-j", - str(self.num_jobs), - ], - env=env, - ) - - def run_tests( - self, - schedule_type, - owner, - test_filter, - test_exclude, - retry: int, - no_testpilot, - timeout=None, - ) -> None: - env = self._compute_env() - ctest = path_search(env, "ctest") - cmake = path_search(env, "cmake") - - def require_command(path: Optional[str], name: str) -> str: - if path is None: - raise RuntimeError("unable to find command `{}`".format(name)) - return path - - # On Windows, we also need to update $PATH to include the directories that - # contain runtime library dependencies. This is not needed on other platforms - # since CMake will emit RPATH properly in the binary so they can find these - # dependencies. - if self.build_opts.is_windows(): - path_entries = self.get_dev_run_extra_path_dirs() - path = env.get("PATH") - if path: - path_entries.insert(0, path) - env["PATH"] = ";".join(path_entries) - - # Don't use the cmd_prefix when running tests. This is vcvarsall.bat on - # Windows. vcvarsall.bat is only needed for the build, not tests. It - # unfortunately fails if invoked with a long PATH environment variable when - # running the tests. - use_cmd_prefix = False - - def get_property(test, propname, defval=None): - """extracts a named property from a cmake test info json blob. - The properties look like: - [{"name": "WORKING_DIRECTORY"}, - {"value": "something"}] - We assume that it is invalid for the same named property to be - listed more than once. - """ - props = test.get("properties", []) - for p in props: - if p.get("name", None) == propname: - return p.get("value", defval) - return defval - - def list_tests(): - output = subprocess.check_output( - [require_command(ctest, "ctest"), "--show-only=json-v1"], - env=env, - cwd=self.build_dir, - ) - try: - data = json.loads(output.decode("utf-8")) - except ValueError as exc: - raise Exception( - "Failed to decode cmake test info using %s: %s. Output was: %r" - % (ctest, str(exc), output) - ) - - tests = [] - machine_suffix = self.build_opts.host_type.as_tuple_string() - for test in data["tests"]: - working_dir = get_property(test, "WORKING_DIRECTORY") - labels = [] - machine_suffix = self.build_opts.host_type.as_tuple_string() - labels.append("tpx-fb-test-type=3") - labels.append("tpx_test_config::buildsystem=getdeps") - labels.append("tpx_test_config::platform={}".format(machine_suffix)) - - if get_property(test, "DISABLED"): - labels.append("disabled") - command = test["command"] - if working_dir: - command = [ - require_command(cmake, "cmake"), - "-E", - "chdir", - working_dir, - ] + command - - import os - - tests.append( - { - "type": "custom", - "target": "%s-%s-getdeps-%s" - % (self.manifest.name, test["name"], machine_suffix), - "command": command, - "labels": labels, - "env": {}, - "required_paths": [], - "contacts": [], - "cwd": os.getcwd(), - } - ) - return tests - - discover_like_continuous = False - if schedule_type == "continuous" or ( - schedule_type == "base_retry" and is_public_commit(self.build_opts) - ): - discover_like_continuous = True - - if discover_like_continuous or schedule_type == "testwarden": - # for continuous and testwarden runs, disabling retry can give up - # better signals for flaky tests. - retry = 0 - - tpx = None - try: - from .facebook.testinfra import start_run - - tpx = path_search(env, "tpx") - except ImportError: - # internal testinfra not available - pass - - if tpx and not no_testpilot: - buck_test_info = list_tests() - import os - - buck_test_info_name = os.path.join(self.build_dir, ".buck-test-info.json") - with open(buck_test_info_name, "w") as f: - json.dump(buck_test_info, f) - - env.set("http_proxy", "") - env.set("https_proxy", "") - runs = [] - - with start_run(env["FBSOURCE_HASH"]) as run_id: - testpilot_args = [ - tpx, - "--force-local-execution", - "--buck-test-info", - buck_test_info_name, - "--retry=%d" % retry, - "-j=%s" % str(self.num_jobs), - "--print-long-results", - ] - - if owner: - testpilot_args += ["--contacts", owner] - - if env: - testpilot_args.append("--env") - testpilot_args.extend(f"{key}={val}" for key, val in env.items()) - - if run_id is not None: - testpilot_args += ["--run-id", run_id] - - if timeout is not None: - testpilot_args += ["--timeout", str(timeout)] - - if test_filter: - testpilot_args += ["--", test_filter] - - if schedule_type == "diff": - runs.append(["--collection", "oss-diff", "--purpose", "diff"]) - elif discover_like_continuous: - runs.append( - [ - "--tag-new-tests", - "--collection", - "oss-continuous", - "--purpose", - "continuous", - ] - ) - elif schedule_type == "testwarden": - # One run to assess new tests - runs.append( - [ - "--tag-new-tests", - "--collection", - "oss-new-test-stress", - "--stress-runs", - "10", - "--purpose", - "stress-run-new-test", - ] - ) - # And another for existing tests - runs.append( - [ - "--tag-new-tests", - "--collection", - "oss-existing-test-stress", - "--stress-runs", - "10", - "--purpose", - "stress-run", - ] - ) - else: - runs.append([]) - - for run in runs: - # FIXME: What is this trying to accomplish? Should it fail on first or >=1 errors? - self._run_cmd( - testpilot_args + run, - cwd=self.build_opts.fbcode_builder_dir, - env=env, - use_cmd_prefix=use_cmd_prefix, - ) - else: - args = [ - require_command(ctest, "ctest"), - "--output-on-failure", - "-j", - str(self.num_jobs), - ] - if test_filter: - args += ["-R", test_filter] - if test_exclude: - args += ["--exclude-regex", test_exclude] - if timeout is not None: - args += ["--timeout", str(timeout)] - - count = 0 - retcode = -1 - while count <= retry: - # FIXME: What is this trying to accomplish? Should it fail on first or >=1 errors? - retcode = self._check_cmd( - args, env=env, use_cmd_prefix=use_cmd_prefix, allow_fail=True - ) - - if retcode == 0: - break - if count == 0: - # Only add this option in the second run. - args += ["--rerun-failed"] - count += 1 - if retcode is not None and retcode != 0: - # Allow except clause in getdeps.main to catch and exit gracefully - # This allows non-testpilot runs to fail through the same logic as failed testpilot runs, which may become handy in case if post test processing is needed in the future - raise subprocess.CalledProcessError(retcode, args) - - -class NinjaBootstrap(BuilderBase): - def __init__( - self, - loader, - dep_manifests, - build_opts, - ctx, - manifest, - build_dir, - src_dir, - inst_dir, - ) -> None: - super(NinjaBootstrap, self).__init__( - loader, - dep_manifests, - build_opts, - ctx, - manifest, - src_dir, - build_dir, - inst_dir, - ) - - def _build(self, reconfigure) -> None: - self._check_cmd( - [sys.executable, "configure.py", "--bootstrap"], cwd=self.src_dir - ) - src_ninja = os.path.join(self.src_dir, "ninja") - dest_ninja = os.path.join(self.inst_dir, "bin/ninja") - bin_dir = os.path.dirname(dest_ninja) - if not os.path.exists(bin_dir): - os.makedirs(bin_dir) - shutil.copyfile(src_ninja, dest_ninja) - shutil.copymode(src_ninja, dest_ninja) - - -class OpenSSLBuilder(BuilderBase): - def __init__( - self, - loader, - dep_manifests, - build_opts, - ctx, - manifest, - build_dir, - src_dir, - inst_dir, - ) -> None: - super(OpenSSLBuilder, self).__init__( - loader, - dep_manifests, - build_opts, - ctx, - manifest, - src_dir, - build_dir, - inst_dir, - ) - - def _build(self, reconfigure) -> None: - configure = os.path.join(self.src_dir, "Configure") - - # prefer to resolve the perl that we installed from - # our manifest on windows, but fall back to the system - # path on eg: darwin - env = self.env.copy() - for m in self.dep_manifests: - bindir = os.path.join(self.loader.get_project_install_dir(m), "bin") - add_path_entry(env, "PATH", bindir, append=False) - - perl = typing.cast(str, path_search(env, "perl", "perl")) - - make_j_args = [] - extra_args = [] - if self.build_opts.is_windows(): - # jom is compatible with nmake, adds the /j argument for parallel build - make = "jom.exe" - make_j_args = ["/j%s" % self.num_jobs] - args = ["VC-WIN64A-masm", "-utf-8"] - # fixes "if multiple CL.EXE write to the same .PDB file, please use /FS" - extra_args = ["/FS"] - elif self.build_opts.is_darwin(): - make = "make" - make_j_args = ["-j%s" % self.num_jobs] - args = ( - ["darwin64-x86_64-cc"] - if not self.build_opts.is_arm() - else ["darwin64-arm64-cc"] - ) - elif self.build_opts.is_linux(): - make = "make" - make_j_args = ["-j%s" % self.num_jobs] - args = ( - ["linux-x86_64"] if not self.build_opts.is_arm() else ["linux-aarch64"] - ) - else: - raise Exception("don't know how to build openssl for %r" % self.ctx) - - self._check_cmd( - [ - perl, - configure, - "--prefix=%s" % self.inst_dir, - "--openssldir=%s" % self.inst_dir, - ] - + args - + [ - "enable-static-engine", - "enable-capieng", - "no-makedepend", - "no-unit-test", - "no-tests", - ] - + extra_args - ) - # show the config produced - self._check_cmd([perl, "configdata.pm", "--dump"], env=env) - make_build = [make] + make_j_args - self._check_cmd(make_build, env=env) - make_install = [make, "install_sw", "install_ssldirs"] - self._check_cmd(make_install, env=env) - - -class Boost(BuilderBase): - def __init__( - self, - loader, - dep_manifests, - build_opts, - ctx, - manifest, - src_dir, - build_dir, - inst_dir, - b2_args, - ) -> None: - children = os.listdir(src_dir) - assert len(children) == 1, "expected a single directory entry: %r" % (children,) - boost_src = children[0] - assert boost_src.startswith("boost") - src_dir = os.path.join(src_dir, children[0]) - super(Boost, self).__init__( - loader, - dep_manifests, - build_opts, - ctx, - manifest, - src_dir, - build_dir, - inst_dir, - ) - self.b2_args = b2_args - - def _build(self, reconfigure) -> None: - env = self._compute_env() - linkage = ["static"] - if self.build_opts.is_windows() or self.build_opts.shared_libs: - linkage.append("shared") - - args = [] - if self.build_opts.is_darwin(): - clang = subprocess.check_output(["xcrun", "--find", "clang"]) - user_config = os.path.join(self.build_dir, "project-config.jam") - with open(user_config, "w") as jamfile: - jamfile.write("using clang : : %s ;\n" % clang.decode().strip()) - args.append("--user-config=%s" % user_config) - - for link in linkage: - bootstrap_args = self.manifest.get_section_as_args( - "bootstrap.args", self.ctx - ) - if self.build_opts.is_windows(): - bootstrap = os.path.join(self.src_dir, "bootstrap.bat") - self._check_cmd([bootstrap] + bootstrap_args, cwd=self.src_dir, env=env) - args += ["address-model=64"] - else: - bootstrap = os.path.join(self.src_dir, "bootstrap.sh") - self._check_cmd( - [bootstrap, "--prefix=%s" % self.inst_dir] + bootstrap_args, - cwd=self.src_dir, - env=env, - ) - - b2 = os.path.join(self.src_dir, "b2") - self._check_cmd( - [ - b2, - "-j%s" % self.num_jobs, - "--prefix=%s" % self.inst_dir, - "--builddir=%s" % self.build_dir, - ] - + args - + self.b2_args - + [ - "link=%s" % link, - "runtime-link=shared", - "variant=release", - "threading=multi", - "debug-symbols=on", - "visibility=global", - "-d2", - "install", - ], - cwd=self.src_dir, - env=env, - ) - - -class NopBuilder(BuilderBase): - def __init__( - self, loader, dep_manifests, build_opts, ctx, manifest, src_dir, inst_dir - ) -> None: - super(NopBuilder, self).__init__( - loader, dep_manifests, build_opts, ctx, manifest, src_dir, None, inst_dir - ) - - def build(self, reconfigure: bool) -> None: - print("Installing %s -> %s" % (self.src_dir, self.inst_dir)) - parent = os.path.dirname(self.inst_dir) - if not os.path.exists(parent): - os.makedirs(parent) - - install_files = self.manifest.get_section_as_ordered_pairs( - "install.files", self.ctx - ) - if install_files: - for src_name, dest_name in self.manifest.get_section_as_ordered_pairs( - "install.files", self.ctx - ): - full_dest = os.path.join(self.inst_dir, dest_name) - full_src = os.path.join(self.src_dir, src_name) - - dest_parent = os.path.dirname(full_dest) - if not os.path.exists(dest_parent): - os.makedirs(dest_parent) - if os.path.isdir(full_src): - if not os.path.exists(full_dest): - simple_copytree(full_src, full_dest) - else: - shutil.copyfile(full_src, full_dest) - shutil.copymode(full_src, full_dest) - # This is a bit gross, but the mac ninja.zip doesn't - # give ninja execute permissions, so force them on - # for things that look like they live in a bin dir - if os.path.dirname(dest_name) == "bin": - st = os.lstat(full_dest) - os.chmod(full_dest, st.st_mode | stat.S_IXUSR) - else: - if not os.path.exists(self.inst_dir): - simple_copytree(self.src_dir, self.inst_dir) - - -class SetupPyBuilder(BuilderBase): - def _build(self, reconfigure) -> None: - env = self._compute_env() - - setup_env = self.manifest.get_section_as_dict("setup-py.env", self.ctx) - for key, value in setup_env.items(): - env[key] = value - - setup_py_path = os.path.join(self.src_dir, "setup.py") - - if not os.path.exists(setup_py_path): - raise RuntimeError(f"setup.py script not found at {setup_py_path}") - - self._check_cmd( - [path_search(env, "python3"), setup_py_path, "install"], - cwd=self.src_dir, - env=env, - ) - - # Create the installation directory if it doesn't exist - os.makedirs(self.inst_dir, exist_ok=True) - - # Mark the project as built - with open(os.path.join(self.inst_dir, ".built-by-getdeps"), "w") as f: - f.write("built") - - def run_tests( - self, - schedule_type, - owner, - test_filter, - test_exclude, - retry, - no_testpilot, - timeout=None, - ) -> None: - # setup.py actually no longer has a standard command for running tests. - # Instead we let manifest files specify an arbitrary Python file to run - # as a test. - - # Get the test command from the manifest - python_script = self.manifest.get( - "setup-py.test", "python_script", ctx=self.ctx - ) - if not python_script: - print(f"No test script specified for {self.manifest.name}") - return - - # Run the command - env = self._compute_env() - self._check_cmd(["python3", python_script], cwd=self.src_dir, env=env) - - -class SqliteBuilder(BuilderBase): - def __init__( - self, - loader, - dep_manifests, - build_opts, - ctx, - manifest, - src_dir, - build_dir, - inst_dir, - ) -> None: - super(SqliteBuilder, self).__init__( - loader, - dep_manifests, - build_opts, - ctx, - manifest, - src_dir, - build_dir, - inst_dir, - ) - - def _build(self, reconfigure) -> None: - for f in ["sqlite3.c", "sqlite3.h", "sqlite3ext.h"]: - src = os.path.join(self.src_dir, f) - dest = os.path.join(self.build_dir, f) - copy_if_different(src, dest) - - cmake_lists = """ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) -project(sqlite3 C) -add_library(sqlite3 STATIC sqlite3.c) -# These options are taken from the defaults in Makefile.msc in -# the sqlite distribution -target_compile_definitions(sqlite3 PRIVATE - -DSQLITE_ENABLE_COLUMN_METADATA=1 - -DSQLITE_ENABLE_FTS3=1 - -DSQLITE_ENABLE_RTREE=1 - -DSQLITE_ENABLE_GEOPOLY=1 - -DSQLITE_ENABLE_JSON1=1 - -DSQLITE_ENABLE_STMTVTAB=1 - -DSQLITE_ENABLE_DBPAGE_VTAB=1 - -DSQLITE_ENABLE_DBSTAT_VTAB=1 - -DSQLITE_INTROSPECTION_PRAGMAS=1 - -DSQLITE_ENABLE_DESERIALIZE=1 -) -install(TARGETS sqlite3) -install(FILES sqlite3.h sqlite3ext.h DESTINATION include) - """ - - with open(os.path.join(self.build_dir, "CMakeLists.txt"), "w") as f: - f.write(cmake_lists) - - defines = { - "CMAKE_INSTALL_PREFIX": self.inst_dir, - "BUILD_SHARED_LIBS": "ON" if self.build_opts.shared_libs else "OFF", - "CMAKE_BUILD_TYPE": "RelWithDebInfo", - } - define_args = ["-D%s=%s" % (k, v) for (k, v) in defines.items()] - define_args += ["-G", "Ninja"] - - env = self._compute_env() - - # Resolve the cmake that we installed - cmake = path_search(env, "cmake") - - self._check_cmd([cmake, self.build_dir] + define_args, env=env) - self._check_cmd( - [ - cmake, - "--build", - self.build_dir, - "--target", - "install", - "--config", - self.build_opts.build_type, - "-j", - str(self.num_jobs), - ], - env=env, - ) diff --git a/build/fbcode_builder/getdeps/buildopts.py b/build/fbcode_builder/getdeps/buildopts.py deleted file mode 100644 index 3002c0758..000000000 --- a/build/fbcode_builder/getdeps/buildopts.py +++ /dev/null @@ -1,699 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-unsafe - -import errno -import glob -import ntpath -import os -import subprocess -import sys -import tempfile -from typing import Mapping, Optional - -from .copytree import containing_repo_type -from .envfuncs import add_flag, add_path_entry, Env -from .fetcher import get_fbsource_repo_data, homebrew_package_prefix -from .manifest import ContextGenerator -from .platform import get_available_ram, HostType, is_windows - - -GITBASH_TMP = "c:\\tools\\fb.gitbash\\tmp" - - -def detect_project(path): - repo_type, repo_root = containing_repo_type(path) - if repo_type is None: - return None, None - - # Look for a .projectid file. If it exists, read the project name from it. - project_id_path = os.path.join(repo_root, ".projectid") - try: - with open(project_id_path, "r") as f: - project_name = f.read().strip() - return repo_root, project_name - except EnvironmentError as ex: - if ex.errno != errno.ENOENT: - raise - - return repo_root, None - - -class BuildOptions(object): - def __init__( - self, - fbcode_builder_dir, - scratch_dir, - host_type, - install_dir=None, - num_jobs: int = 0, - use_shipit: bool = False, - vcvars_path=None, - allow_system_packages: bool = False, - lfs_path=None, - shared_libs: bool = False, - facebook_internal=None, - free_up_disk: bool = False, - build_type: Optional[str] = None, - ) -> None: - """fbcode_builder_dir - the path to either the in-fbsource fbcode_builder dir, - or for shipit-transformed repos, the build dir that - has been mapped into that dir. - scratch_dir - a place where we can store repos and build bits. - This path should be stable across runs and ideally - should not be in the repo of the project being built, - but that is ultimately where we generally fall back - for builds outside of FB - install_dir - where the project will ultimately be installed - num_jobs - the level of concurrency to use while building - use_shipit - use real shipit instead of the simple shipit transformer - vcvars_path - Path to external VS toolchain's vsvarsall.bat - shared_libs - whether to build shared libraries - free_up_disk - take extra actions to save runner disk space - build_type - CMAKE_BUILD_TYPE, used by cmake and cargo builders - """ - - if not install_dir: - install_dir = os.path.join(scratch_dir, "installed") - - self.project_hashes = None - for p in ["../deps/github_hashes", "../project_hashes"]: - hashes = os.path.join(fbcode_builder_dir, p) - if os.path.exists(hashes): - self.project_hashes = hashes - break - - # Detect what repository and project we are being run from. - self.repo_root, self.repo_project = detect_project(os.getcwd()) - - # If we are running from an fbsource repository, set self.fbsource_dir - # to allow the ShipIt-based fetchers to use it. - if self.repo_project == "fbsource": - self.fbsource_dir: Optional[str] = self.repo_root - else: - self.fbsource_dir = None - - if facebook_internal is None: - if self.fbsource_dir: - facebook_internal = True - else: - facebook_internal = False - - self.facebook_internal = facebook_internal - self.specified_num_jobs = num_jobs - self.scratch_dir = scratch_dir - self.install_dir = install_dir - self.fbcode_builder_dir = fbcode_builder_dir - self.host_type = host_type - self.use_shipit = use_shipit - self.allow_system_packages = allow_system_packages - self.lfs_path = lfs_path - self.shared_libs = shared_libs - self.free_up_disk = free_up_disk - - if build_type is None: - build_type = "RelWithDebInfo" - - self.build_type = build_type - - lib_path = None - if self.is_darwin(): - lib_path = "DYLD_LIBRARY_PATH" - elif self.is_linux(): - lib_path = "LD_LIBRARY_PATH" - elif self.is_windows(): - lib_path = "PATH" - else: - lib_path = None - self.lib_path = lib_path - - if vcvars_path is None and is_windows(): - - try: - # Allow a site-specific vcvarsall path. - from .facebook.vcvarsall import build_default_vcvarsall - except ImportError: - vcvarsall = [] - else: - vcvarsall = ( - build_default_vcvarsall(self.fbsource_dir) - if self.fbsource_dir is not None - else [] - ) - - # On Windows, the compiler is not available in the PATH by - # default so we need to run the vcvarsall script to populate the - # environment. We use a glob to find some version of this script - # as deployed with Visual Studio. - if len(vcvarsall) == 0: - # check the 64 bit installs - for year in ["2022"]: - vcvarsall += glob.glob( - os.path.join( - os.environ.get("ProgramFiles", "C:\\Program Files"), - "Microsoft Visual Studio", - year, - "*", - "VC", - "Auxiliary", - "Build", - "vcvarsall.bat", - ) - ) - - # then the 32 bit ones - for year in ["2022", "2019", "2017"]: - vcvarsall += glob.glob( - os.path.join( - os.environ["ProgramFiles(x86)"], - "Microsoft Visual Studio", - year, - "*", - "VC", - "Auxiliary", - "Build", - "vcvarsall.bat", - ) - ) - if len(vcvarsall) == 0: - raise Exception( - "Could not find vcvarsall.bat. Please install Visual Studio." - ) - vcvars_path = vcvarsall[0] - print(f"Using vcvarsall.bat from {vcvars_path}", file=sys.stderr) - - self.vcvars_path = vcvars_path - - @property - def manifests_dir(self): - return os.path.join(self.fbcode_builder_dir, "manifests") - - def is_darwin(self): - return self.host_type.is_darwin() - - def is_windows(self): - return self.host_type.is_windows() - - def is_arm(self): - return self.host_type.is_arm() - - def get_vcvars_path(self): - return self.vcvars_path - - def is_linux(self): - return self.host_type.is_linux() - - def is_freebsd(self): - return self.host_type.is_freebsd() - - def get_num_jobs(self, job_weight: int) -> int: - """Given an estimated job_weight in MiB, compute a reasonable concurrency limit.""" - if self.specified_num_jobs: - return self.specified_num_jobs - - available_ram = get_available_ram() - - import multiprocessing - - return max(1, min(multiprocessing.cpu_count(), available_ram // job_weight)) - - def get_context_generator(self, host_tuple=None): - """Create a manifest ContextGenerator for the specified target platform.""" - if host_tuple is None: - host_type = self.host_type - elif isinstance(host_tuple, HostType): - host_type = host_tuple - else: - host_type = HostType.from_tuple_string(host_tuple) - - return ContextGenerator( - { - "os": host_type.ostype, - "distro": host_type.distro, - "distro_vers": host_type.distrovers, - "fb": "on" if self.facebook_internal else "off", - "fbsource": "on" if self.fbsource_dir else "off", - "test": "off", - "shared_libs": "on" if self.shared_libs else "off", - } - ) - - def compute_env_for_install_dirs( - self, loader, dep_manifests, ctx, env=None, manifest=None - ): # noqa: C901 - if env is not None: - env = env.copy() - else: - env = Env() - - env["GETDEPS_BUILD_DIR"] = os.path.join(self.scratch_dir, "build") - env["GETDEPS_INSTALL_DIR"] = self.install_dir - - # Python setuptools attempts to discover a local MSVC for - # building Python extensions. On Windows, getdeps already - # supports invoking a vcvarsall prior to compilation. - # - # Tell setuptools to bypass its own search. This fixes a bug - # where setuptools would fail when run from CMake on GitHub - # Actions with the inscrutable message 'error: Microsoft - # Visual C++ 14.0 is required. Get it with "Build Tools for - # Visual Studio"'. I suspect the actual error is that the - # environment or PATH is overflowing. - # - # For extra credit, someone could patch setuptools to - # propagate the actual error message from vcvarsall, because - # often it does not mean Visual C++ is not available. - # - # Related discussions: - # - https://github.com/pypa/setuptools/issues/2028 - # - https://github.com/pypa/setuptools/issues/2307 - # - https://developercommunity.visualstudio.com/t/error-microsoft-visual-c-140-is-required/409173 - # - https://github.com/OpenMS/OpenMS/pull/4779 - # - https://github.com/actions/virtual-environments/issues/1484 - - if self.is_windows() and self.get_vcvars_path(): - env["DISTUTILS_USE_SDK"] = "1" - - # On macOS we need to set `SDKROOT` when we use clang for system - # header files. - if self.is_darwin() and "SDKROOT" not in env: - sdkroot = subprocess.check_output(["xcrun", "--show-sdk-path"]) - env["SDKROOT"] = sdkroot.decode().strip() - - if ( - self.is_darwin() - and self.allow_system_packages - and self.host_type.get_package_manager() == "homebrew" - and manifest - and manifest.resolved_system_packages - ): - # Homebrew packages may not be on the default PATHs - brew_packages = manifest.resolved_system_packages.get("homebrew", []) - for p in brew_packages: - found = self.add_homebrew_package_to_env(p, env) - # Try extra hard to find openssl, needed with homebrew on macOS - if found and p.startswith("openssl"): - candidate = homebrew_package_prefix("openssl@1.1") - if os.path.exists(candidate): - os.environ["OPENSSL_ROOT_DIR"] = candidate - env["OPENSSL_ROOT_DIR"] = os.environ["OPENSSL_ROOT_DIR"] - - if self.fbsource_dir: - env["YARN_YARN_OFFLINE_MIRROR"] = os.path.join( - self.fbsource_dir, "xplat/third-party/yarn/offline-mirror" - ) - yarn_exe = "yarn.bat" if self.is_windows() else "yarn" - env["YARN_PATH"] = os.path.join( - self.fbsource_dir, "xplat/third-party/yarn/", yarn_exe - ) - node_exe = "node-win-x64.exe" if self.is_windows() else "node" - env["NODE_BIN"] = os.path.join( - self.fbsource_dir, "xplat/third-party/node/bin/", node_exe - ) - env["RUST_VENDORED_CRATES_DIR"] = os.path.join( - self.fbsource_dir, "third-party/rust/vendor" - ) - hash_data = get_fbsource_repo_data(self) - env["FBSOURCE_HASH"] = hash_data.hash - env["FBSOURCE_DATE"] = hash_data.date - - # reverse as we are prepending to the PATHs - for m in reversed(dep_manifests): - is_direct_dep = ( - manifest is not None and m.name in manifest.get_dependencies(ctx) - ) - d = loader.get_project_install_dir(m) - if os.path.exists(d): - self.add_prefix_to_env( - d, - env, - append=False, - is_direct_dep=is_direct_dep, - ) - - # Linux is always system openssl - system_openssl = self.is_linux() - - # For other systems lets see if package is requested - if not system_openssl and manifest and manifest.resolved_system_packages: - for _pkg_type, pkgs in manifest.resolved_system_packages.items(): - for p in pkgs: - if p.startswith("openssl") or p.startswith("libssl"): - system_openssl = True - break - - # Let openssl know to pick up the system certs if present - if system_openssl or "OPENSSL_DIR" in env: - for system_ssl_cfg in ["/etc/pki/tls", "/etc/ssl"]: - if os.path.isdir(system_ssl_cfg): - cert_dir = system_ssl_cfg + "/certs" - if os.path.isdir(cert_dir): - env["SSL_CERT_DIR"] = cert_dir - cert_file = system_ssl_cfg + "/cert.pem" - if os.path.isfile(cert_file): - env["SSL_CERT_FILE"] = cert_file - - return env - - def add_homebrew_package_to_env(self, package, env) -> bool: - prefix = homebrew_package_prefix(package) - if prefix and os.path.exists(prefix): - return self.add_prefix_to_env( - prefix, env, append=False, add_library_path=True - ) - return False - - def add_prefix_to_env( - self, - d, - env, - append: bool = True, - add_library_path: bool = False, - is_direct_dep: bool = False, - ) -> bool: # noqa: C901 - bindir = os.path.join(d, "bin") - found = False - has_pkgconfig = False - pkgconfig = os.path.join(d, "lib", "pkgconfig") - if os.path.exists(pkgconfig): - found = True - has_pkgconfig = True - add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig, append=append) - - pkgconfig = os.path.join(d, "lib64", "pkgconfig") - if os.path.exists(pkgconfig): - found = True - has_pkgconfig = True - add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig, append=append) - - add_path_entry(env, "CMAKE_PREFIX_PATH", d, append=append) - - # Tell the thrift compiler about includes it needs to consider - thriftdir = os.path.join(d, "include", "thrift-files") - if os.path.exists(thriftdir): - found = True - add_path_entry(env, "THRIFT_INCLUDE_PATH", thriftdir, append=append) - - # module detection for python is old fashioned and needs flags - includedir = os.path.join(d, "include") - if os.path.exists(includedir): - found = True - ncursesincludedir = os.path.join(d, "include", "ncurses") - if os.path.exists(ncursesincludedir): - add_path_entry(env, "C_INCLUDE_PATH", ncursesincludedir, append=append) - add_flag(env, "CPPFLAGS", f"-I{includedir}", append=append) - add_flag(env, "CPPFLAGS", f"-I{ncursesincludedir}", append=append) - elif "/bz2-" in d: - add_flag(env, "CPPFLAGS", f"-I{includedir}", append=append) - # For non-pkgconfig projects Cabal has no way to find the includes or - # libraries, so we provide a set of extra Cabal flags in the env - if not has_pkgconfig and is_direct_dep: - add_flag( - env, - "GETDEPS_CABAL_FLAGS", - f"--extra-include-dirs={includedir}", - append=append, - ) - - # The thrift compiler's built-in includes are installed directly to the include dir - includethriftdir = os.path.join(d, "include", "thrift") - if os.path.exists(includethriftdir): - add_path_entry(env, "THRIFT_INCLUDE_PATH", includedir, append=append) - - # Map from FB python manifests to PYTHONPATH - pydir = os.path.join(d, "lib", "fb-py-libs") - if os.path.exists(pydir): - found = True - manifest_ext = ".manifest" - pymanifestfiles = [ - f - for f in os.listdir(pydir) - if f.endswith(manifest_ext) and os.path.isfile(os.path.join(pydir, f)) - ] - for f in pymanifestfiles: - subdir = f[: -len(manifest_ext)] - add_path_entry( - env, "PYTHONPATH", os.path.join(pydir, subdir), append=append - ) - - # Allow resolving shared objects built earlier (eg: zstd - # doesn't include the full path to the dylib in its linkage - # so we need to give it an assist) - if self.lib_path: - for lib in ["lib", "lib64"]: - libdir = os.path.join(d, lib) - if os.path.exists(libdir): - found = True - add_path_entry(env, self.lib_path, libdir, append=append) - # module detection for python is old fashioned and needs flags - if "/ncurses-" in d: - add_flag(env, "LDFLAGS", f"-L{libdir}", append=append) - elif "/bz2-" in d: - add_flag(env, "LDFLAGS", f"-L{libdir}", append=append) - if add_library_path: - add_path_entry(env, "LIBRARY_PATH", libdir, append=append) - if not has_pkgconfig and is_direct_dep: - add_flag( - env, - "GETDEPS_CABAL_FLAGS", - f"--extra-lib-dirs={libdir}", - append=append, - ) - - # Allow resolving binaries (eg: cmake, ninja) and dlls - # built by earlier steps - if os.path.exists(bindir): - found = True - add_path_entry(env, "PATH", bindir, append=append) - - # If rustc is present in the `bin` directory, set RUSTC to prevent - # cargo uses the rustc installed in the system. - if self.is_windows(): - cargo_path = os.path.join(bindir, "cargo.exe") - rustc_path = os.path.join(bindir, "rustc.exe") - rustdoc_path = os.path.join(bindir, "rustdoc.exe") - else: - cargo_path = os.path.join(bindir, "cargo") - rustc_path = os.path.join(bindir, "rustc") - rustdoc_path = os.path.join(bindir, "rustdoc") - - if os.path.isfile(rustc_path): - env["CARGO_BIN"] = cargo_path - env["RUSTC"] = rustc_path - env["RUSTDOC"] = rustdoc_path - - openssl_include = os.path.join(d, "include", "openssl") - if os.path.isdir(openssl_include) and any( - os.path.isfile(os.path.join(d, "lib", libcrypto)) - for libcrypto in ("libcrypto.lib", "libcrypto.so", "libcrypto.a") - ): - # This must be the openssl library, let Rust know about it - env["OPENSSL_DIR"] = d - - return found - - -def list_win32_subst_letters(): - output = subprocess.check_output(["subst"]).decode("utf-8") - # The output is a set of lines like: `F:\: => C:\open\some\where` - lines = output.strip().split("\r\n") - mapping = {} - for line in lines: - fields = line.split(": => ") - if len(fields) != 2: - continue - letter = fields[0] - path = fields[1] - mapping[letter] = path - - return mapping - - -def find_existing_win32_subst_for_path( - path: str, - subst_mapping: Mapping[str, str], -) -> Optional[str]: - path = ntpath.normcase(ntpath.normpath(path)) - for letter, target in subst_mapping.items(): - if ntpath.normcase(target) == path: - return letter - return None - - -def find_unused_drive_letter(): - import ctypes - - buffer_len = 256 - blen = ctypes.c_uint(buffer_len) - rv = ctypes.c_uint() - bufs = ctypes.create_string_buffer(buffer_len) - rv = ctypes.windll.kernel32.GetLogicalDriveStringsA(blen, bufs) - if rv > buffer_len: - raise Exception("GetLogicalDriveStringsA result too large for buffer") - nul = "\x00".encode("ascii") - - used = [drive.decode("ascii")[0] for drive in bufs.raw.strip(nul).split(nul)] - possible = [c for c in "ABCDEFGHIJKLMNOPQRSTUVWXYZ"] - available = sorted(list(set(possible) - set(used))) - if len(available) == 0: - return None - # Prefer to assign later letters rather than earlier letters - return available[-1] - - -def map_subst_path(path: str) -> str: - """find a short drive letter mapping for a path""" - for _attempt in range(0, 24): - drive = find_existing_win32_subst_for_path( - path, subst_mapping=list_win32_subst_letters() - ) - if drive: - return drive - available = find_unused_drive_letter() - if available is None: - raise Exception( - ( - "unable to make shorter subst mapping for %s; " - "no available drive letters" - ) - % path - ) - - # Try to set up a subst mapping; note that we may be racing with - # other processes on the same host, so this may not succeed. - try: - subprocess.check_call(["subst", "%s:" % available, path]) - subst = "%s:\\" % available - print("Mapped scratch dir %s -> %s" % (path, subst), file=sys.stderr) - return subst - except Exception: - print("Failed to map %s -> %s" % (available, path), file=sys.stderr) - - raise Exception("failed to set up a subst path for %s" % path) - - -def _check_host_type(args, host_type): - if host_type is None: - host_tuple_string = getattr(args, "host_type", None) - if host_tuple_string: - host_type = HostType.from_tuple_string(host_tuple_string) - else: - host_type = HostType() - - assert isinstance(host_type, HostType) - return host_type - - -def setup_build_options(args, host_type=None) -> BuildOptions: - """Create a BuildOptions object based on the arguments""" - - fbcode_builder_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - scratch_dir = args.scratch_path - if not scratch_dir: - # TODO: `mkscratch` doesn't currently know how best to place things on - # sandcastle, so whip up something reasonable-ish - if "SANDCASTLE" in os.environ: - if "DISK_TEMP" not in os.environ: - raise Exception( - ( - "I need DISK_TEMP to be set in the sandcastle environment " - "so that I can store build products somewhere sane" - ) - ) - - disk_temp = os.environ["DISK_TEMP"] - if is_windows(): - # force use gitbash tmp dir for windows, as its less likely to have a tmp cleaner - # that removes extracted prior dated source files - os.makedirs(GITBASH_TMP, exist_ok=True) - print( - f"Using {GITBASH_TMP} instead of DISK_TEMP {disk_temp} for scratch dir", - file=sys.stderr, - ) - disk_temp = GITBASH_TMP - - scratch_dir = os.path.join(disk_temp, "fbcode_builder_getdeps") - if not scratch_dir: - try: - scratch_dir = ( - subprocess.check_output( - ["mkscratch", "path", "--subdir", "fbcode_builder_getdeps"] - ) - .strip() - .decode("utf-8") - ) - except OSError as exc: - if exc.errno != errno.ENOENT: - # A legit failure; don't fall back, surface the error - raise - # This system doesn't have mkscratch so we fall back to - # something local. - munged = fbcode_builder_dir.replace("Z", "zZ") - for s in ["/", "\\", ":"]: - munged = munged.replace(s, "Z") - - if is_windows() and os.path.isdir("c:/open"): - temp = "c:/open/scratch" - else: - temp = tempfile.gettempdir() - - scratch_dir = os.path.join(temp, "fbcode_builder_getdeps-%s" % munged) - if not is_windows() and os.geteuid() == 0: - # Running as root; in the case where someone runs - # sudo getdeps.py install-system-deps - # and then runs as build without privs, we want to avoid creating - # a scratch dir that the second stage cannot write to. - # So we generate a different path if we are root. - scratch_dir += "-root" - - if not os.path.exists(scratch_dir): - os.makedirs(scratch_dir) - - if is_windows(): - subst = map_subst_path(scratch_dir) - scratch_dir = subst - else: - if not os.path.exists(scratch_dir): - os.makedirs(scratch_dir) - - # Make sure we normalize the scratch path. This path is used as part of the hash - # computation for detecting if projects have been updated, so we need to always - # use the exact same string to refer to a given directory. - # But! realpath in some combinations of Windows/Python3 versions can expand the - # drive substitutions on Windows, so avoid that! - if not is_windows(): - scratch_dir = os.path.realpath(scratch_dir) - - # Save these args passed by the user in an env variable, so it - # can be used while hashing this build. - os.environ["GETDEPS_CMAKE_DEFINES"] = getattr(args, "extra_cmake_defines", "") or "" - - host_type = _check_host_type(args, host_type) - - build_args = { - k: v - for (k, v) in vars(args).items() - if k - in { - "num_jobs", - "use_shipit", - "vcvars_path", - "allow_system_packages", - "lfs_path", - "shared_libs", - "free_up_disk", - "build_type", - } - } - - return BuildOptions( - fbcode_builder_dir, - scratch_dir, - host_type, - install_dir=args.install_prefix, - facebook_internal=args.facebook_internal, - **build_args, - ) diff --git a/build/fbcode_builder/getdeps/cache.py b/build/fbcode_builder/getdeps/cache.py deleted file mode 100644 index ed0d45bfd..000000000 --- a/build/fbcode_builder/getdeps/cache.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-unsafe - - -class ArtifactCache(object): - """The ArtifactCache is a small abstraction that allows caching - named things in some external storage mechanism. - The primary use case is for storing the build products on CI - systems to accelerate the build""" - - def download_to_file(self, name, dest_file_name) -> bool: - """If `name` exists in the cache, download it and place it - in the specified `dest_file_name` location on the filesystem. - If a transient issue was encountered a TransientFailure shall - be raised. - If `name` doesn't exist in the cache `False` shall be returned. - If `dest_file_name` was successfully updated `True` shall be - returned. - All other conditions shall raise an appropriate exception.""" - return False - - def upload_from_file(self, name, source_file_name) -> None: - """Causes `name` to be populated in the cache by uploading - the contents of `source_file_name` to the storage system. - If a transient issue was encountered a TransientFailure shall - be raised. - If the upload failed for some other reason, an appropriate - exception shall be raised.""" - pass - - -def create_cache() -> None: - """This function is monkey patchable to provide an actual - implementation""" - return None diff --git a/build/fbcode_builder/getdeps/cargo.py b/build/fbcode_builder/getdeps/cargo.py deleted file mode 100644 index e2ab432bc..000000000 --- a/build/fbcode_builder/getdeps/cargo.py +++ /dev/null @@ -1,506 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-unsafe - -import os -import re -import shutil -import sys -import typing - -from .builder import BuilderBase -from .copytree import rmtree_more, simple_copytree - -if typing.TYPE_CHECKING: - from .buildopts import BuildOptions - - -class CargoBuilder(BuilderBase): - def __init__( - self, - loader, - dep_manifests, # manifests of dependencies - build_opts: "BuildOptions", - ctx, - manifest, - src_dir, - build_dir, - inst_dir, - build_doc, - workspace_dir, - manifests_to_build, - cargo_config_file, - ) -> None: - super(CargoBuilder, self).__init__( - loader, - dep_manifests, - build_opts, - ctx, - manifest, - src_dir, - build_dir, - inst_dir, - ) - self.build_doc = build_doc - self.ws_dir = workspace_dir - self.manifests_to_build = manifests_to_build and manifests_to_build.split(",") - self.loader = loader - self.cargo_config_file_subdir = cargo_config_file - - def run_cargo(self, install_dirs, operation, args=None) -> None: - args = args or [] - env = self._compute_env() - # Enable using nightly features with stable compiler - env["RUSTC_BOOTSTRAP"] = "1" - env["LIBZ_SYS_STATIC"] = "1" - cmd = [ - "cargo", - operation, - "--workspace", - "-j%s" % self.num_jobs, - ] + args - self._check_cmd(cmd, cwd=self.workspace_dir(), env=env) - - def build_source_dir(self): - return os.path.join(self.build_dir, "source") - - def workspace_dir(self): - return os.path.join(self.build_source_dir(), self.ws_dir or "") - - def manifest_dir(self, manifest): - return os.path.join(self.build_source_dir(), manifest) - - def recreate_dir(self, src, dst) -> None: - if os.path.isdir(dst): - if os.path.islink(dst): - os.remove(dst) - else: - rmtree_more(dst) - simple_copytree(src, dst) - - def recreate_linked_dir(self, src, dst) -> None: - if os.path.isdir(dst): - if os.path.islink(dst): - os.remove(dst) - elif os.path.isdir(dst): - shutil.rmtree(dst) - os.symlink(src, dst) - - def cargo_config_file(self): - build_source_dir = self.build_dir - if self.cargo_config_file_subdir: - return os.path.join(build_source_dir, self.cargo_config_file_subdir) - else: - return os.path.join(build_source_dir, ".cargo", "config.toml") - - def _create_cargo_config(self): - cargo_config_file = self.cargo_config_file() - cargo_config_dir = os.path.dirname(cargo_config_file) - if not os.path.isdir(cargo_config_dir): - os.mkdir(cargo_config_dir) - - dep_to_git = self._resolve_dep_to_git() - - if os.path.isfile(cargo_config_file): - with open(cargo_config_file, "r") as f: - print(f"Reading {cargo_config_file}", file=sys.stderr) - cargo_content = f.read() - else: - cargo_content = "" - - new_content = cargo_content - if "# Generated by getdeps.py" not in cargo_content: - new_content += """\ -# Generated by getdeps.py -[build] -target-dir = '''{}''' - -[profile.dev] -debug = false -incremental = false - -[profile.release] -opt-level = "{}" -""".format( - self.build_dir.replace("\\", "\\\\"), - "z" if self.build_opts.build_type == "MinSizeRel" else "s", - ) - - # Point to vendored sources from getdeps manifests - for _dep, git_conf in dep_to_git.items(): - if "cargo_vendored_sources" in git_conf: - vendored_dir = git_conf["cargo_vendored_sources"].replace("\\", "\\\\") - override = ( - f'[source."{git_conf["repo_url"]}"]\ndirectory = "{vendored_dir}"\n' - ) - if override not in cargo_content: - new_content += override - - if self.build_opts.fbsource_dir: - # Point to vendored crates.io if possible - try: - from .facebook.rust import vendored_crates - - new_content = vendored_crates( - self.build_opts.fbsource_dir, new_content - ) - except ImportError: - # This FB internal module isn't shippped to github, - # so just rely on cargo downloading crates on it's own - pass - - if new_content != cargo_content: - with open(cargo_config_file, "w") as f: - print( - f"Writing cargo config for {self.manifest.name} to {cargo_config_file}", - file=sys.stderr, - ) - f.write(new_content) - - return dep_to_git - - def _prepare(self, reconfigure) -> None: - build_source_dir = self.build_source_dir() - self.recreate_dir(self.src_dir, build_source_dir) - - dep_to_git = self._create_cargo_config() - - if self.ws_dir is not None: - self._patchup_workspace(dep_to_git) - - def _build(self, reconfigure) -> None: - # _prepare has been run already. Actually do the build - build_source_dir = self.build_source_dir() - - build_args = [ - "--artifact-dir", - os.path.join(self.inst_dir, "bin"), - "-Zunstable-options", - ] - - if self.build_opts.build_type != "Debug": - build_args.append("--release") - - if self.manifests_to_build is None: - self.run_cargo( - self.install_dirs, - "build", - build_args, - ) - else: - for manifest in self.manifests_to_build: - self.run_cargo( - self.install_dirs, - "build", - build_args - + [ - "--manifest-path", - self.manifest_dir(manifest), - ], - ) - - self.recreate_linked_dir( - build_source_dir, os.path.join(self.inst_dir, "source") - ) - - def run_tests( - self, - schedule_type, - owner, - test_filter, - test_exclude, - retry, - no_testpilot, - timeout=None, - ) -> None: - build_args = [] - if self.build_opts.build_type != "Debug": - build_args.append("--release") - - if test_filter: - filter_args = ["--", test_filter] - else: - filter_args = [] - - if self.manifests_to_build is None: - self.run_cargo(self.install_dirs, "test", build_args + filter_args) - if self.build_doc and not filter_args: - self.run_cargo(self.install_dirs, "doc", ["--no-deps"]) - else: - for manifest in self.manifests_to_build: - margs = ["--manifest-path", self.manifest_dir(manifest)] - self.run_cargo( - self.install_dirs, "test", build_args + filter_args + margs - ) - if self.build_doc and not filter_args: - self.run_cargo(self.install_dirs, "doc", ["--no-deps"] + margs) - - def _patchup_workspace(self, dep_to_git) -> None: - """ - This method makes some assumptions about the state of the project and - its cargo dependendies: - 1. Crates from cargo dependencies can be extracted from Cargo.toml files - using _extract_crates function. It is using a heuristic so check its - code to understand how it is done. - 2. The extracted cargo dependencies crates can be found in the - dependency's install dir using _resolve_crate_to_path function - which again is using a heuristic. - - Notice that many things might go wrong here. E.g. if someone depends - on another getdeps crate by writing in their Cargo.toml file: - - my-rename-of-crate = { package = "crate", git = "..." } - - they can count themselves lucky because the code will raise an - Exception. There might be more cases where the code will silently pass - producing bad results. - """ - workspace_dir = self.workspace_dir() - git_url_to_crates_and_paths = self._resolve_config(dep_to_git) - if git_url_to_crates_and_paths: - patch_cargo = os.path.join(workspace_dir, "Cargo.toml") - if os.path.isfile(patch_cargo): - with open(patch_cargo, "r") as f: - manifest_content = f.read() - else: - manifest_content = "" - - new_content = manifest_content - if "[package]" not in manifest_content: - # A fake manifest has to be crated to change the virtual - # manifest into a non-virtual. The virtual manifests are limited - # in many ways and the inability to define patches on them is - # one. Check https://github.com/rust-lang/cargo/issues/4934 to - # see if it is resolved. - null_file = "/dev/null" - if self.build_opts.is_windows(): - null_file = "nul" - new_content += f""" -[package] -name = "fake_manifest_of_{self.manifest.name}" -version = "0.0.0" - -[lib] -path = "{null_file}" -""" - config = [] - for git_url, crates_to_patch_path in git_url_to_crates_and_paths.items(): - crates_patches = [ - '{} = {{ path = "{}" }}'.format( - crate, - crates_to_patch_path[crate].replace("\\", "\\\\"), - ) - for crate in sorted(crates_to_patch_path.keys()) - ] - patch_key = f'[patch."{git_url}"]' - if patch_key not in manifest_content: - config.append(f"\n{patch_key}\n" + "\n".join(crates_patches)) - new_content += "\n".join(config) - if new_content != manifest_content: - with open(patch_cargo, "w") as f: - print( - f"writing patch to {patch_cargo}", - file=sys.stderr, - ) - f.write(new_content) - - def _resolve_config(self, dep_to_git) -> typing.Dict[str, typing.Dict[str, str]]: - """ - Returns a configuration to be put inside root Cargo.toml file which - patches the dependencies git code with local getdeps versions. - See https://doc.rust-lang.org/cargo/reference/manifest.html#the-patch-section - """ - dep_to_crates = self._resolve_dep_to_crates(self.build_source_dir(), dep_to_git) - - git_url_to_crates_and_paths = {} - for dep_name in sorted(dep_to_git.keys()): - git_conf = dep_to_git[dep_name] - req_crates = sorted(dep_to_crates.get(dep_name, [])) - if not req_crates: - continue # nothing to patch, move along - - git_url = git_conf.get("repo_url", None) - crate_source_map = git_conf["crate_source_map"] - if git_url and crate_source_map: - crates_to_patch_path = git_url_to_crates_and_paths.get(git_url, {}) - for c in req_crates: - if c in crate_source_map and c not in crates_to_patch_path: - crates_to_patch_path[c] = crate_source_map[c] - print( - f"{self.manifest.name}: Patching crate {c} via virtual manifest in {self.workspace_dir()}", - file=sys.stderr, - ) - if crates_to_patch_path: - git_url_to_crates_and_paths[git_url] = crates_to_patch_path - - return git_url_to_crates_and_paths - - def _resolve_dep_to_git(self): - """ - For each direct dependency of the currently build manifest check if it - is also cargo-builded and if yes then extract it's git configs and - install dir - """ - dependencies = self.manifest.get_dependencies(self.ctx) - if not dependencies: - return [] - - dep_to_git = {} - for dep in dependencies: - dep_manifest = self.loader.load_manifest(dep) - dep_builder = dep_manifest.get("build", "builder", ctx=self.ctx) - - dep_cargo_conf = dep_manifest.get_section_as_dict("cargo", self.ctx) - dep_crate_map = dep_manifest.get_section_as_dict("crate.pathmap", self.ctx) - - if ( - not (dep_crate_map or dep_cargo_conf) - and dep_builder not in ["cargo"] - or dep == "rust" - ): - # This dependency has no cargo rust content so ignore it. - # The "rust" dependency is an exception since it contains the - # toolchain. - continue - - git_conf = dep_manifest.get_section_as_dict("git", self.ctx) - if dep != "rust" and "repo_url" not in git_conf: - raise Exception( - f"{dep}: A cargo dependency requires git.repo_url to be defined." - ) - - if dep_builder == "cargo": - dep_source_dir = self.loader.get_project_install_dir(dep_manifest) - dep_source_dir = os.path.join(dep_source_dir, "source") - else: - fetcher = self.loader.create_fetcher(dep_manifest) - dep_source_dir = fetcher.get_src_dir() - - crate_source_map = {} - if dep_crate_map: - for crate, subpath in dep_crate_map.items(): - if crate not in crate_source_map: - if self.build_opts.is_windows(): - subpath = subpath.replace("/", "\\") - crate_path = os.path.join(dep_source_dir, subpath) - print( - f"{self.manifest.name}: Mapped crate {crate} to dep {dep} dir {crate_path}", - file=sys.stderr, - ) - crate_source_map[crate] = crate_path - elif dep_cargo_conf: - # We don't know what crates are defined buy the dep, look for them - search_pattern = re.compile('\\[package\\]\nname = "(.*)"') - for crate_root, _, files in os.walk(dep_source_dir): - if "Cargo.toml" in files: - with open(os.path.join(crate_root, "Cargo.toml"), "r") as f: - content = f.read() - match = search_pattern.search(content) - if match: - crate = match.group(1) - if crate: - print( - f"{self.manifest.name}: Discovered crate {crate} in dep {dep} dir {crate_root}", - file=sys.stderr, - ) - crate_source_map[crate] = crate_root - - git_conf["crate_source_map"] = crate_source_map - - if not dep_crate_map and dep_cargo_conf: - dep_cargo_dir = self.loader.get_project_build_dir(dep_manifest) - dep_cargo_dir = os.path.join(dep_cargo_dir, "source") - dep_ws_dir = dep_cargo_conf.get("workspace_dir", None) - if dep_ws_dir: - dep_cargo_dir = os.path.join(dep_cargo_dir, dep_ws_dir) - git_conf["cargo_vendored_sources"] = dep_cargo_dir - - dep_to_git[dep] = git_conf - return dep_to_git - - def _resolve_dep_to_crates(self, build_source_dir, dep_to_git): - """ - This function traverse the build_source_dir in search of Cargo.toml - files, extracts the crate names from them using _extract_crates - function and returns a merged result containing crate names per - dependency name from all Cargo.toml files in the project. - """ - if not dep_to_git: - return {} # no deps, so don't waste time traversing files - - dep_to_crates = {} - - # First populate explicit crate paths from dependencies - for name, git_conf in dep_to_git.items(): - crates = git_conf["crate_source_map"].keys() - if crates: - dep_to_crates.setdefault(name, set()).update(crates) - - # Now find from Cargo.tomls - for root, _, files in os.walk(build_source_dir): - for f in files: - if f == "Cargo.toml": - more_dep_to_crates = CargoBuilder._extract_crates_used( - os.path.join(root, f), dep_to_git - ) - for dep_name, crates in more_dep_to_crates.items(): - existing_crates = dep_to_crates.get(dep_name, set()) - for c in crates: - if c not in existing_crates: - print( - f"Patch {self.manifest.name} uses {dep_name} crate {crates}", - file=sys.stderr, - ) - existing_crates.add(c) - dep_to_crates.setdefault(name, set()).update(existing_crates) - return dep_to_crates - - @staticmethod - def _extract_crates_used(cargo_toml_file, dep_to_git): - """ - This functions reads content of provided cargo toml file and extracts - crate names per each dependency. The extraction is done by a heuristic - so it might be incorrect. - """ - deps_to_crates = {} - with open(cargo_toml_file, "r") as f: - for line in f.readlines(): - if line.startswith("#") or "git = " not in line: - continue # filter out commented lines and ones without git deps - for dep_name, conf in dep_to_git.items(): - # Only redirect deps that point to git URLS - if 'git = "{}"'.format(conf["repo_url"]) in line: - pkg_template = ' package = "' - if pkg_template in line: - crate_name, _, _ = line.partition(pkg_template)[ - 2 - ].partition('"') - else: - crate_name, _, _ = line.partition("=") - deps_to_crates.setdefault(dep_name, set()).add( - crate_name.strip() - ) - return deps_to_crates - - def _resolve_crate_to_path(self, crate, crate_source_map): - """ - Tries to find in source_dir by searching a [package] - keyword followed by name = "". - """ - search_pattern = '[package]\nname = "{}"'.format(crate) - - for _crate, crate_source_dir in crate_source_map.items(): - for crate_root, _, files in os.walk(crate_source_dir): - if "Cargo.toml" in files: - with open(os.path.join(crate_root, "Cargo.toml"), "r") as f: - content = f.read() - if search_pattern in content: - return crate_root - - raise Exception( - f"{self.manifest.name}: Failed to find dep crate {crate} in paths {crate_source_map}" - ) diff --git a/build/fbcode_builder/getdeps/copytree.py b/build/fbcode_builder/getdeps/copytree.py deleted file mode 100644 index 491c373e4..000000000 --- a/build/fbcode_builder/getdeps/copytree.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-unsafe - -import os -import shutil -import stat -import subprocess - -from .platform import is_windows -from .runcmd import run_cmd - - -PREFETCHED_DIRS = set() - - -def containing_repo_type(path): - while True: - if os.path.exists(os.path.join(path, ".git")): - return ("git", path) - if os.path.exists(os.path.join(path, ".hg")): - return ("hg", path) - - parent = os.path.dirname(path) - if parent == path: - return None, None - path = parent - - -def find_eden_root(dirpath): - """If the specified directory is inside an EdenFS checkout, returns - the canonical absolute path to the root of that checkout. - - Returns None if the specified directory is not in an EdenFS checkout. - """ - if is_windows(): - repo_type, repo_root = containing_repo_type(dirpath) - if repo_root is not None: - if os.path.exists(os.path.join(repo_root, ".eden", "config")): - return repo_root - return None - - try: - return os.readlink(os.path.join(dirpath, ".eden", "root")) - except OSError: - return None - - -def prefetch_dir_if_eden(dirpath) -> None: - """After an amend/rebase, Eden may need to fetch a large number - of trees from the servers. The simplistic single threaded walk - performed by copytree makes this more expensive than is desirable - so we help accelerate things by performing a prefetch on the - source directory""" - global PREFETCHED_DIRS - if dirpath in PREFETCHED_DIRS: - return - root = find_eden_root(dirpath) - if root is None: - return - glob = f"{os.path.relpath(dirpath, root).replace(os.sep, '/')}/**" - print(f"Prefetching {glob}") - subprocess.call(["edenfsctl", "prefetch", "--repo", root, glob, "--background"]) - PREFETCHED_DIRS.add(dirpath) - - -def simple_copytree(src_dir, dest_dir, symlinks=False): - """A simple version of shutil.copytree() that can delegate to native tools if faster""" - if is_windows(): - os.makedirs(dest_dir, exist_ok=True) - cmd = [ - "robocopy.exe", - src_dir, - dest_dir, - # copy directories, including empty ones - "/E", - # Ignore Extra files in destination - "/XX", - # enable parallel copy - "/MT", - # be quiet - "/NFL", - "/NDL", - "/NJH", - "/NJS", - "/NP", - ] - if symlinks: - cmd.append("/SL") - # robocopy exits with code 1 if it copied ok, hence allow_fail - # https://learn.microsoft.com/en-us/troubleshoot/windows-server/backup-and-storage/return-codes-used-robocopy-utility - exit_code = run_cmd(cmd, allow_fail=True) - if exit_code > 1: - raise subprocess.CalledProcessError(exit_code, cmd) - return dest_dir - else: - return shutil.copytree(src_dir, dest_dir, symlinks=symlinks) - - -def _remove_readonly_and_try_again(func, path, exc_info): - """ - Error handler for shutil.rmtree. - If the error is due to an access error (read only file) - it attempts to add write permission and then retries the operation. - Any other failure propagates. - """ - # exc_info is a tuple (exc_type, exc_value, traceback) - exc_type = exc_info[0] - if exc_type is PermissionError: - os.chmod(path, stat.S_IWRITE) - # Retry the original function (os.remove or os.rmdir) - try: - func(path) - except Exception: - # If it still fails, the original exception from func() will propagate - raise - else: - # If the error is not a PermissionError, re-raise the original exception - raise exc_info[1] - - -def rmtree_more(path): - """Wrapper around shutil.rmtree() that makes it remove readonly files as well. - Useful when git on windows decides to make some files readonly on checkout""" - shutil.rmtree(path, onerror=_remove_readonly_and_try_again) diff --git a/build/fbcode_builder/getdeps/dyndeps.py b/build/fbcode_builder/getdeps/dyndeps.py deleted file mode 100644 index 9b79131a9..000000000 --- a/build/fbcode_builder/getdeps/dyndeps.py +++ /dev/null @@ -1,467 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-unsafe - -import errno -import glob -import os -import re -import shlex -import shutil -import stat -import subprocess -import sys -from struct import unpack -from typing import List, Optional - -OBJECT_SUBDIRS = ("bin", "lib", "lib64") - - -def copyfile(src, dest) -> None: - shutil.copyfile(src, dest) - shutil.copymode(src, dest) - - -class DepBase(object): - def __init__(self, buildopts, env, install_dirs, strip) -> None: - self.buildopts = buildopts - self.env = env - self.install_dirs = install_dirs - self.strip = strip - - # Deduplicates dependency processing. Keyed on the library - # destination path. - self.processed_deps = set() - - def list_dynamic_deps(self, objfile): - raise RuntimeError("list_dynamic_deps not implemented") - - def interesting_dep(self, d) -> bool: - return True - - # final_install_prefix must be the equivalent path to `destdir` on the - # installed system. For example, if destdir is `/tmp/RANDOM/usr/local' which - # is intended to map to `/usr/local` in the install image, then - # final_install_prefix='/usr/local'. - # If left unspecified, destdir will be used. - def process_deps(self, destdir, final_install_prefix=None) -> None: - if self.buildopts.is_windows(): - lib_dir = "bin" - else: - lib_dir = "lib" - # pyre-fixme[16]: `DepBase` has no attribute `munged_lib_dir`. - self.munged_lib_dir = os.path.join(destdir, lib_dir) - - final_lib_dir = os.path.join(final_install_prefix or destdir, lib_dir) - - if not os.path.isdir(self.munged_lib_dir): - os.makedirs(self.munged_lib_dir) - - # Look only at the things that got installed in the leaf package, - # which will be the last entry in the install dirs list - inst_dir = self.install_dirs[-1] - print("Process deps under %s" % inst_dir, file=sys.stderr) - - for dir in OBJECT_SUBDIRS: - src_dir = os.path.join(inst_dir, dir) - if not os.path.isdir(src_dir): - continue - dest_dir = os.path.join(destdir, dir) - if not os.path.exists(dest_dir): - os.makedirs(dest_dir) - - for objfile in self.list_objs_in_dir(src_dir): - print("Consider %s/%s" % (dir, objfile)) - dest_obj = os.path.join(dest_dir, objfile) - copyfile(os.path.join(src_dir, objfile), dest_obj) - self.munge_in_place(dest_obj, final_lib_dir) - - def find_all_dependencies(self, build_dir): - all_deps = set() - for objfile in self.list_objs_in_dir( - build_dir, recurse=True, output_prefix=build_dir - ): - for d in self.list_dynamic_deps(objfile): - all_deps.add(d) - - interesting_deps = {d for d in all_deps if self.interesting_dep(d)} - dep_paths = [] - for dep in interesting_deps: - dep_path = self.resolve_loader_path(dep) - if dep_path: - dep_paths.append(dep_path) - - return dep_paths - - def munge_in_place(self, objfile, final_lib_dir) -> None: - print("Munging %s" % objfile) - for d in self.list_dynamic_deps(objfile): - if not self.interesting_dep(d): - continue - - # Resolve this dep: does it exist in any of our installation - # directories? If so, then it is a candidate for processing - dep = self.resolve_loader_path(d) - if dep: - # pyre-fixme[16]: `DepBase` has no attribute `munged_lib_dir`. - dest_dep = os.path.join(self.munged_lib_dir, os.path.basename(dep)) - print("dep: %s -> %s" % (d, dest_dep)) - if dest_dep in self.processed_deps: - # A previous dependency with the same name has already - # been installed at dest_dep, so there is no need to copy - # or munge the dependency again. - # TODO: audit that both source paths have the same inode number - pass - else: - self.processed_deps.add(dest_dep) - copyfile(dep, dest_dep) - self.munge_in_place(dest_dep, final_lib_dir) - - self.rewrite_dep(objfile, d, dep, dest_dep, final_lib_dir) - - if self.strip: - self.strip_debug_info(objfile) - - def rewrite_dep(self, objfile, depname, old_dep, new_dep, final_lib_dir): - raise RuntimeError("rewrite_dep not implemented") - - def resolve_loader_path(self, dep: str) -> Optional[str]: - if os.path.isabs(dep): - return dep - d = os.path.basename(dep) - for inst_dir in self.install_dirs: - for libdir in OBJECT_SUBDIRS: - candidate = os.path.join(inst_dir, libdir, d) - if os.path.exists(candidate): - return candidate - return None - - def list_objs_in_dir(self, dir, recurse: bool = False, output_prefix: str = ""): - for entry in os.listdir(dir): - entry_path = os.path.join(dir, entry) - st = os.lstat(entry_path) - if stat.S_ISREG(st.st_mode): - if self.is_objfile(entry_path): - relative_result = os.path.join(output_prefix, entry) - yield os.path.normcase(relative_result) - elif recurse and stat.S_ISDIR(st.st_mode): - child_prefix = os.path.join(output_prefix, entry) - for result in self.list_objs_in_dir( - entry_path, recurse=recurse, output_prefix=child_prefix - ): - yield result - - def is_objfile(self, objfile) -> bool: - return True - - def strip_debug_info(self, objfile) -> None: - """override this to define how to remove debug information - from an object file""" - pass - - def check_call_verbose(self, args: List[str]) -> None: - print(" ".join(map(shlex.quote, args))) - subprocess.check_call(args) - - -class WinDeps(DepBase): - def __init__(self, buildopts, env, install_dirs, strip) -> None: - super(WinDeps, self).__init__(buildopts, env, install_dirs, strip) - self.dumpbin = self.find_dumpbin() - - def find_dumpbin(self) -> str: - # Looking for dumpbin in the following hardcoded paths. - # The registry option to find the install dir doesn't work anymore. - globs = [ - ( - "C:/Program Files/" - "Microsoft Visual Studio/" - "*/*/VC/Tools/" - "MSVC/*/bin/Hostx64/x64/dumpbin.exe" - ), - ( - "C:/Program Files (x86)/" - "Microsoft Visual Studio/" - "*/*/VC/Tools/" - "MSVC/*/bin/Hostx64/x64/dumpbin.exe" - ), - ( - "C:/Program Files (x86)/" - "Common Files/" - "Microsoft/Visual C++ for Python/*/" - "VC/bin/dumpbin.exe" - ), - ("c:/Program Files (x86)/Microsoft Visual Studio */VC/bin/dumpbin.exe"), - ( - "C:/Program Files/Microsoft Visual Studio/*/Professional/VC/Tools/MSVC/*/bin/HostX64/x64/dumpbin.exe" - ), - ] - for pattern in globs: - for exe in glob.glob(pattern): - return exe - - raise RuntimeError("could not find dumpbin.exe") - - def list_dynamic_deps(self, exe): - deps = [] - print("Resolve deps for %s" % exe) - output = subprocess.check_output( - [self.dumpbin, "/nologo", "/dependents", exe] - ).decode("utf-8") - - lines = output.split("\n") - for line in lines: - m = re.match("\\s+(\\S+.dll)", line, re.IGNORECASE) - if m: - deps.append(m.group(1).lower()) - - return deps - - def rewrite_dep(self, objfile, depname, old_dep, new_dep, final_lib_dir) -> None: - # We can't rewrite on windows, but we will - # place the deps alongside the exe so that - # they end up in the search path - pass - - # These are the Windows system dll, which we don't want to copy while - # packaging. - SYSTEM_DLLS = set( # noqa: C405 - [ - "advapi32.dll", - "dbghelp.dll", - "kernel32.dll", - "msvcp140.dll", - "vcruntime140.dll", - "ws2_32.dll", - "ntdll.dll", - "shlwapi.dll", - ] - ) - - def interesting_dep(self, d) -> bool: - if "api-ms-win-crt" in d: - return False - if d in self.SYSTEM_DLLS: - return False - return True - - def is_objfile(self, objfile) -> bool: - if not os.path.isfile(objfile): - return False - if objfile.lower().endswith(".exe"): - return True - return False - - def emit_dev_run_script(self, script_path, dep_dirs) -> None: - """Emit a script that can be used to run build artifacts directly from the - build directory, without installing them. - - The dep_dirs parameter should be a list of paths that need to be added to $PATH. - This can be computed by calling compute_dependency_paths() or - compute_dependency_paths_fast(). - - This is only necessary on Windows, which does not have RPATH, and instead - requires the $PATH environment variable be updated in order to find the proper - library dependencies. - """ - contents = self._get_dev_run_script_contents(dep_dirs) - with open(script_path, "w") as f: - f.write(contents) - - def compute_dependency_paths(self, build_dir): - """Return a list of all directories that need to be added to $PATH to ensure - that library dependencies can be found correctly. This is computed by scanning - binaries to determine exactly the right list of dependencies. - - The compute_dependency_paths_fast() is a alternative function that runs faster - but may return additional extraneous paths. - """ - dep_dirs = set() - # Find paths by scanning the binaries. - for dep in self.find_all_dependencies(build_dir): - dep_dirs.add(os.path.dirname(dep)) - - dep_dirs.update(self.read_custom_dep_dirs(build_dir)) - return sorted(dep_dirs) - - def compute_dependency_paths_fast(self, build_dir): - """Similar to compute_dependency_paths(), but rather than actually scanning - binaries, just add all library paths from the specified installation - directories. This is much faster than scanning the binaries, but may result in - more paths being returned than actually necessary. - """ - dep_dirs = set() - for inst_dir in self.install_dirs: - for subdir in OBJECT_SUBDIRS: - path = os.path.join(inst_dir, subdir) - if os.path.exists(path): - dep_dirs.add(path) - - dep_dirs.update(self.read_custom_dep_dirs(build_dir)) - return sorted(dep_dirs) - - def read_custom_dep_dirs(self, build_dir): - # The build system may also have included libraries from other locations that - # we might not be able to find normally in find_all_dependencies(). - # To handle this situation we support reading additional library paths - # from a LIBRARY_DEP_DIRS.txt file that may have been generated in the build - # output directory. - dep_dirs = set() - try: - explicit_dep_dirs_path = os.path.join(build_dir, "LIBRARY_DEP_DIRS.txt") - with open(explicit_dep_dirs_path, "r") as f: - for line in f.read().splitlines(): - dep_dirs.add(line) - except OSError as ex: - if ex.errno != errno.ENOENT: - raise - - return dep_dirs - - def _get_dev_run_script_contents(self, path_dirs) -> str: - path_entries = ["$env:PATH"] + path_dirs - path_str = ";".join(path_entries) - return """\ -$orig_env = $env:PATH -$env:PATH = "{path_str}" - -try {{ - $cmd_args = $args[1..$args.length] - & $args[0] @cmd_args -}} finally {{ - $env:PATH = $orig_env -}} -""".format( - path_str=path_str - ) - - -class ElfDeps(DepBase): - def __init__(self, buildopts, env, install_dirs, strip) -> None: - super(ElfDeps, self).__init__(buildopts, env, install_dirs, strip) - - # We need patchelf to rewrite deps, so ensure that it is built... - args = [sys.executable, sys.argv[0]] - if buildopts.allow_system_packages: - args.append("--allow-system-packages") - subprocess.check_call(args + ["build", "patchelf"]) - - # ... and that we know where it lives - patchelf_install = os.fsdecode( - subprocess.check_output(args + ["show-inst-dir", "patchelf"]).strip() - ) - if not patchelf_install: - # its a system package, so we assume it is in the path - patchelf_install = "patchelf" - else: - patchelf_install = os.path.join(patchelf_install, "bin", "patchelf") - self.patchelf = patchelf_install - - def list_dynamic_deps(self, objfile): - out = ( - subprocess.check_output( - [self.patchelf, "--print-needed", objfile], env=dict(self.env.items()) - ) - .decode("utf-8") - .strip() - ) - lines = out.split("\n") - return lines - - def rewrite_dep(self, objfile, depname, old_dep, new_dep, final_lib_dir) -> None: - final_dep = os.path.join( - final_lib_dir, - # pyre-fixme[16]: `ElfDeps` has no attribute `munged_lib_dir`. - os.path.relpath(new_dep, self.munged_lib_dir), - ) - self.check_call_verbose( - [self.patchelf, "--replace-needed", depname, final_dep, objfile] - ) - - def is_objfile(self, objfile) -> bool: - if not os.path.isfile(objfile): - return False - with open(objfile, "rb") as f: - # https://en.wikipedia.org/wiki/Executable_and_Linkable_Format#File_header - magic = f.read(4) - return magic == b"\x7fELF" - - def strip_debug_info(self, objfile) -> None: - self.check_call_verbose(["strip", objfile]) - - -# MACH-O magic number -MACH_MAGIC = 0xFEEDFACF - - -class MachDeps(DepBase): - def interesting_dep(self, d) -> bool: - if d.startswith("/usr/lib/") or d.startswith("/System/"): - return False - return True - - def is_objfile(self, objfile): - if not os.path.isfile(objfile): - return False - with open(objfile, "rb") as f: - # mach stores the magic number in native endianness, - # so unpack as native here and compare - header = f.read(4) - if len(header) != 4: - return False - magic = unpack("I", header)[0] - return magic == MACH_MAGIC - - def list_dynamic_deps(self, objfile): - if not self.interesting_dep(objfile): - return - out = ( - subprocess.check_output( - ["otool", "-L", objfile], env=dict(self.env.items()) - ) - .decode("utf-8") - .strip() - ) - lines = out.split("\n") - deps = [] - for line in lines: - m = re.match("\t(\\S+)\\s", line) - if m: - if os.path.basename(m.group(1)) != os.path.basename(objfile): - deps.append(os.path.normcase(m.group(1))) - return deps - - def rewrite_dep(self, objfile, depname, old_dep, new_dep, final_lib_dir) -> None: - if objfile.endswith(".dylib"): - # Erase the original location from the id of the shared - # object. It doesn't appear to hurt to retain it, but - # it does look weird, so let's rewrite it to be sure. - self.check_call_verbose( - ["install_name_tool", "-id", os.path.basename(objfile), objfile] - ) - final_dep = os.path.join( - final_lib_dir, - # pyre-fixme[16]: `MachDeps` has no attribute `munged_lib_dir`. - os.path.relpath(new_dep, self.munged_lib_dir), - ) - - self.check_call_verbose( - ["install_name_tool", "-change", depname, final_dep, objfile] - ) - - -def create_dyn_dep_munger( - buildopts, env, install_dirs, strip: bool = False -) -> Optional[DepBase]: - if buildopts.is_linux(): - return ElfDeps(buildopts, env, install_dirs, strip) - if buildopts.is_darwin(): - return MachDeps(buildopts, env, install_dirs, strip) - if buildopts.is_windows(): - return WinDeps(buildopts, env, install_dirs, strip) - if buildopts.is_freebsd(): - return ElfDeps(buildopts, env, install_dirs, strip) - return None diff --git a/build/fbcode_builder/getdeps/envfuncs.py b/build/fbcode_builder/getdeps/envfuncs.py deleted file mode 100644 index f32418c93..000000000 --- a/build/fbcode_builder/getdeps/envfuncs.py +++ /dev/null @@ -1,198 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-unsafe - -import os -import shlex -import sys -from typing import Optional - - -class Env(object): - def __init__(self, src=None) -> None: - self._dict = {} - if src is None: - self.update(os.environ) - else: - self.update(src) - - def update(self, src) -> None: - for k, v in src.items(): - self.set(k, v) - - def copy(self) -> "Env": - return Env(self._dict) - - def _key(self, key): - # The `str` cast may not appear to be needed, but without it we run - # into issues when passing the environment to subprocess. The main - # issue is that in python2 `os.environ` (which is the initial source - # of data for the environment) uses byte based strings, but this - # project uses `unicode_literals`. `subprocess` will raise an error - # if the environment that it is passed has a mixture of byte and - # unicode strings. - # It is simplest to force everything to be `str` for the sake of - # consistency. - key = str(key) - if sys.platform.startswith("win"): - # Windows env var names are case insensitive but case preserving. - # An implementation of PAR files on windows gets confused if - # the env block contains keys with conflicting case, so make a - # pass over the contents to remove any. - # While this O(n) scan is technically expensive and gross, it - # is practically not a problem because the volume of calls is - # relatively low and the cost of manipulating the env is dwarfed - # by the cost of spawning a process on windows. In addition, - # since the processes that we run are expensive anyway, this - # overhead is not the worst thing to worry about. - for k in list(self._dict.keys()): - if str(k).lower() == key.lower(): - return k - elif key in self._dict: - return key - return None - - def get(self, key, defval=None): - key = self._key(key) - if key is None: - return defval - return self._dict[key] - - def __getitem__(self, key): - val = self.get(key) - if key is None: - raise KeyError(key) - return val - - def unset(self, key) -> None: - if key is None: - raise KeyError("attempting to unset env[None]") - - key = self._key(key) - if key: - del self._dict[key] - - def __delitem__(self, key) -> None: - self.unset(key) - - def __repr__(self): - return repr(self._dict) - - def set(self, key, value) -> None: - if key is None: - raise KeyError("attempting to assign env[None] = %r" % value) - - if value is None: - raise ValueError("attempting to assign env[%s] = None" % key) - - # The `str` conversion is important to avoid triggering errors - # with subprocess if we pass in a unicode value; see commentary - # in the `_key` method. - key = str(key) - value = str(value) - - # The `unset` call is necessary on windows where the keys are - # case insensitive. Since this dict is case sensitive, simply - # assigning the value to the new key is not sufficient to remove - # the old value. The `unset` call knows how to match keys and - # remove any potential duplicates. - self.unset(key) - self._dict[key] = value - - def __setitem__(self, key, value) -> None: - self.set(key, value) - - def __iter__(self): - return self._dict.__iter__() - - def __len__(self) -> int: - return len(self._dict) - - def keys(self): - return self._dict.keys() - - def values(self): - return self._dict.values() - - def items(self): - return self._dict.items() - - -def add_path_entry( - env, name, item, append: bool = True, separator: str = os.pathsep -) -> None: - """Cause `item` to be added to the path style env var named - `name` held in the `env` dict. `append` specifies whether - the item is added to the end (the default) or should be - prepended if `name` already exists.""" - val = env.get(name, "") - if len(val) > 0: - val = val.split(separator) - else: - val = [] - if append: - val.append(item) - else: - val.insert(0, item) - env.set(name, separator.join(val)) - - -def add_flag(env, name, flag: str, append: bool = True) -> None: - """Cause `flag` to be added to the CXXFLAGS-style env var named - `name` held in the `env` dict. `append` specifies whether the - flag is added to the end (the default) or should be prepended if - `name` already exists.""" - val = shlex.split(env.get(name, "")) - if append: - val.append(flag) - else: - val.insert(0, flag) - env.set(name, " ".join(val)) - - -_path_search_cache = {} -_not_found = object() - - -def tpx_path() -> str: - return "xplat/testinfra/tpx/ctp.tpx" - - -def path_search(env, exename: str, defval: Optional[str] = None) -> Optional[str]: - """Search for exename in the PATH specified in env. - exename is eg: `ninja` and this function knows to append a .exe - to the end on windows. - Returns the path to the exe if found, or None if either no - PATH is set in env or no executable is found.""" - - path = env.get("PATH", None) - if path is None: - return defval - - # The project hash computation code searches for C++ compilers (g++, clang, etc) - # repeatedly. Cache the result so we don't end up searching for these over and over - # again. - cache_key = (path, exename) - result = _path_search_cache.get(cache_key, _not_found) - if result is _not_found: - result = _perform_path_search(path, exename) - _path_search_cache[cache_key] = result - return result - - -def _perform_path_search(path, exename: str) -> Optional[str]: - is_win = sys.platform.startswith("win") - if is_win: - exename = "%s.exe" % exename - - for bindir in path.split(os.pathsep): - full_name = os.path.join(bindir, exename) - if os.path.exists(full_name) and os.path.isfile(full_name): - if not is_win and not os.access(full_name, os.X_OK): - continue - return full_name - - return None diff --git a/build/fbcode_builder/getdeps/errors.py b/build/fbcode_builder/getdeps/errors.py deleted file mode 100644 index 1d01ad0ec..000000000 --- a/build/fbcode_builder/getdeps/errors.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-unsafe - - -class TransientFailure(Exception): - """Raising this error causes getdeps to return with an error code - that Sandcastle will consider to be a retryable transient - infrastructure error""" - - pass - - -class ManifestNotFound(Exception): - def __init__(self, manifest_name) -> None: - super(Exception, self).__init__("Unable to find manifest '%s'" % manifest_name) diff --git a/build/fbcode_builder/getdeps/expr.py b/build/fbcode_builder/getdeps/expr.py deleted file mode 100644 index 3b3d2d13d..000000000 --- a/build/fbcode_builder/getdeps/expr.py +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-unsafe - -import re -import shlex - - -def parse_expr(expr_text, valid_variables): - """parses the simple criteria expression syntax used in - dependency specifications. - Returns an ExprNode instance that can be evaluated like this: - - ``` - expr = parse_expr("os=windows") - ok = expr.eval({ - "os": "windows" - }) - ``` - - Whitespace is allowed between tokens. The following terms - are recognized: - - KEY = VALUE # Evaluates to True if ctx[KEY] == VALUE - not(EXPR) # Evaluates to True if EXPR evaluates to False - # and vice versa - all(EXPR1, EXPR2, ...) # Evaluates True if all of the supplied - # EXPR's also evaluate True - any(EXPR1, EXPR2, ...) # Evaluates True if any of the supplied - # EXPR's also evaluate True, False if - # none of them evaluated true. - """ - - p = Parser(expr_text, valid_variables) - return p.parse() - - -class ExprNode(object): - def eval(self, ctx) -> bool: - return False - - -class TrueExpr(ExprNode): - def eval(self, ctx) -> bool: - return True - - def __str__(self) -> str: - return "true" - - -class NotExpr(ExprNode): - def __init__(self, node) -> None: - self._node = node - - def eval(self, ctx) -> bool: - return not self._node.eval(ctx) - - def __str__(self) -> str: - return "not(%s)" % self._node - - -class AllExpr(ExprNode): - def __init__(self, nodes) -> None: - self._nodes = nodes - - def eval(self, ctx) -> bool: - for node in self._nodes: - if not node.eval(ctx): - return False - return True - - def __str__(self) -> str: - items = [] - for node in self._nodes: - items.append(str(node)) - return "all(%s)" % ",".join(items) - - -class AnyExpr(ExprNode): - def __init__(self, nodes) -> None: - self._nodes = nodes - - def eval(self, ctx) -> bool: - for node in self._nodes: - if node.eval(ctx): - return True - return False - - def __str__(self) -> str: - items = [] - for node in self._nodes: - items.append(str(node)) - return "any(%s)" % ",".join(items) - - -class EqualExpr(ExprNode): - def __init__(self, key, value) -> None: - self._key = key - self._value = value - - def eval(self, ctx): - return ctx.get(self._key) == self._value - - def __str__(self) -> str: - return "%s=%s" % (self._key, self._value) - - -class Parser(object): - def __init__(self, text, valid_variables) -> None: - self.text = text - self.lex = shlex.shlex(text) - self.valid_variables = valid_variables - - def parse(self): - expr = self.top() - garbage = self.lex.get_token() - if garbage != "": - raise Exception( - "Unexpected token %s after EqualExpr in %s" % (garbage, self.text) - ) - return expr - - def top(self): - name = self.ident() - op = self.lex.get_token() - - if op == "(": - parsers = { - "not": self.parse_not, - "any": self.parse_any, - "all": self.parse_all, - } - func = parsers.get(name) - if not func: - raise Exception("invalid term %s in %s" % (name, self.text)) - return func() - - if op == "=": - if name not in self.valid_variables: - raise Exception("unknown variable %r in expression" % (name,)) - # remove shell quote from value so can test things with period in them, e.g "18.04" - unquoted = " ".join(shlex.split(self.lex.get_token())) - return EqualExpr(name, unquoted) - - raise Exception( - "Unexpected token sequence '%s %s' in %s" % (name, op, self.text) - ) - - def ident(self) -> str: - ident = self.lex.get_token() - # pyre-fixme[6]: For 2nd argument expected `str` but got `Optional[str]`. - if not re.match("[a-zA-Z]+", ident): - raise Exception("expected identifier found %s" % ident) - # pyre-fixme[7]: Expected `str` but got `Optional[str]`. - return ident - - def parse_not(self) -> NotExpr: - node = self.top() - expr = NotExpr(node) - tok = self.lex.get_token() - if tok != ")": - raise Exception("expected ')' found %s" % tok) - return expr - - def parse_any(self) -> AnyExpr: - nodes = [] - while True: - nodes.append(self.top()) - tok = self.lex.get_token() - if tok == ")": - break - if tok != ",": - raise Exception("expected ',' or ')' but found %s" % tok) - return AnyExpr(nodes) - - def parse_all(self) -> AllExpr: - nodes = [] - while True: - nodes.append(self.top()) - tok = self.lex.get_token() - if tok == ")": - break - if tok != ",": - raise Exception("expected ',' or ')' but found %s" % tok) - return AllExpr(nodes) diff --git a/build/fbcode_builder/getdeps/fetcher.py b/build/fbcode_builder/getdeps/fetcher.py deleted file mode 100644 index ddad43aa5..000000000 --- a/build/fbcode_builder/getdeps/fetcher.py +++ /dev/null @@ -1,1064 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-unsafe - -import errno -import hashlib -import os -import random -import re -import shlex -import shutil -import stat -import subprocess -import sys -import tarfile -import time -import zipfile -from datetime import datetime -from typing import Dict, NamedTuple -from urllib.parse import urlparse -from urllib.request import Request, urlopen - -from .copytree import prefetch_dir_if_eden -from .envfuncs import Env -from .errors import TransientFailure -from .platform import is_windows -from .runcmd import run_cmd - - -def file_name_is_cmake_file(file_name): - file_name = file_name.lower() - base = os.path.basename(file_name) - return ( - base.endswith(".cmake") - or base.endswith(".cmake.in") - or base == "cmakelists.txt" - ) - - -class ChangeStatus(object): - """Indicates the nature of changes that happened while updating - the source directory. There are two broad uses: - * When extracting archives for third party software we want to - know that we did something (eg: we either extracted code or - we didn't do anything) - * For 1st party code where we use shipit to transform the code, - we want to know if we changed anything so that we can perform - a build, but we generally want to be a little more nuanced - and be able to distinguish between just changing a source file - and whether we might need to reconfigure the build system. - """ - - def __init__(self, all_changed: bool = False) -> None: - """Construct a ChangeStatus object. The default is to create - a status that indicates no changes, but passing all_changed=True - will create one that indicates that everything changed""" - if all_changed: - self.source_files = 1 - self.make_files = 1 - else: - self.source_files = 0 - self.make_files = 0 - - def record_change(self, file_name) -> None: - """Used by the shipit fetcher to record changes as it updates - files in the destination. If the file name might be one used - in the cmake build system that we use for 1st party code, then - record that as a "make file" change. We could broaden this - to match any file used by various build systems, but it is - only really useful for our internal cmake stuff at this time. - If the file isn't a build file and is under the `fbcode_builder` - dir then we don't class that as an interesting change that we - might need to rebuild, so we ignore it. - Otherwise we record the file as a source file change.""" - - file_name = file_name.lower() - if file_name_is_cmake_file(file_name): - self.make_files += 1 - elif "/fbcode_builder/cmake" in file_name: - self.source_files += 1 - elif "/fbcode_builder/" not in file_name: - self.source_files += 1 - - def sources_changed(self) -> bool: - """Returns true if any source files were changed during - an update operation. This will typically be used to decide - that the build system to be run on the source dir in an - incremental mode""" - return self.source_files > 0 - - def build_changed(self) -> bool: - """Returns true if any build files were changed during - an update operation. This will typically be used to decidfe - that the build system should be reconfigured and re-run - as a full build""" - return self.make_files > 0 - - -class Fetcher(object): - """The Fetcher is responsible for fetching and extracting the - sources for project. The Fetcher instance defines where the - extracted data resides and reports this to the consumer via - its `get_src_dir` method.""" - - def update(self) -> ChangeStatus: - """Brings the src dir up to date, ideally minimizing - changes so that a subsequent build doesn't over-build. - Returns a ChangeStatus object that helps the caller to - understand the nature of the changes required during - the update.""" - return ChangeStatus() - - def clean(self) -> None: - """Reverts any changes that might have been made to - the src dir""" - pass - - def hash(self) -> None: - """Returns a hash that identifies the version of the code in the - working copy. For a git repo this is commit hash for the working - copy. For other Fetchers this should relate to the version of - the code in the src dir. The intent is that if a manifest - changes the version/rev of a project that the hash be different. - Importantly, this should be computable without actually fetching - the code, as we want this to factor into a hash used to download - a pre-built version of the code, without having to first download - and extract its sources (eg: boost on windows is pretty painful). - """ - pass - - def get_src_dir(self) -> None: - """Returns the source directory that the project was - extracted into""" - pass - - -class LocalDirFetcher(object): - """This class exists to override the normal fetching behavior, and - use an explicit user-specified directory for the project sources. - - This fetcher cannot update or track changes. It always reports that the - project has changed, forcing it to always be built.""" - - def __init__(self, path) -> None: - self.path = os.path.realpath(path) - - def update(self) -> ChangeStatus: - return ChangeStatus(all_changed=True) - - def hash(self) -> str: - return "0" * 40 - - def get_src_dir(self): - return self.path - - def clean(self) -> None: - pass - - -class SystemPackageFetcher(object): - def __init__(self, build_options, packages) -> None: - self.manager = build_options.host_type.get_package_manager() - self.packages = packages.get(self.manager) - self.host_type = build_options.host_type - if self.packages: - self.installed = None - else: - self.installed = False - - def packages_are_installed(self): - if self.installed is not None: - return self.installed - - cmd = None - if self.manager == "rpm": - cmd = ["rpm", "-q"] + sorted(self.packages) - elif self.manager == "deb": - cmd = ["dpkg", "-s"] + sorted(self.packages) - elif self.manager == "homebrew": - cmd = ["brew", "ls", "--versions"] + sorted(self.packages) - - if cmd: - proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - if proc.returncode == 0: - # captured as binary as we will hash this later - self.installed = proc.stdout - else: - # Need all packages to be present to consider us installed - self.installed = False - - else: - self.installed = False - - return bool(self.installed) - - def update(self) -> ChangeStatus: - assert self.installed - return ChangeStatus(all_changed=False) - - def hash(self) -> str: - if self.packages_are_installed(): - return hashlib.sha256(self.installed).hexdigest() - else: - return "0" * 40 - - def get_src_dir(self) -> None: - return None - - -class PreinstalledNopFetcher(SystemPackageFetcher): - def __init__(self) -> None: - self.installed = True - - -class GitFetcher(Fetcher): - DEFAULT_DEPTH = 1 - - def __init__(self, build_options, manifest, repo_url, rev, depth, branch) -> None: - # Extract the host/path portions of the URL and generate a flattened - # directory name. eg: - # github.com/facebook/folly.git -> github.com-facebook-folly.git - url = urlparse(repo_url) - directory = "%s%s%s" % (url.netloc, url.path, branch if branch else "") - for s in ["/", "\\", ":"]: - directory = directory.replace(s, "-") - - # Place it in a repos dir in the scratch space - repos_dir = os.path.join(build_options.scratch_dir, "repos") - if not os.path.exists(repos_dir): - os.makedirs(repos_dir) - self.repo_dir = os.path.join(repos_dir, directory) - - if not rev and build_options.project_hashes: - hash_file = os.path.join( - build_options.project_hashes, - re.sub("\\.git$", "-rev.txt", url.path[1:]), - ) - if os.path.exists(hash_file): - with open(hash_file, "r") as f: - data = f.read() - m = re.match("Subproject commit ([a-fA-F0-9]{40})", data) - if not m: - raise Exception("Failed to parse rev from %s" % hash_file) - rev = m.group(1) - print( - "Using pinned rev %s for %s" % (rev, repo_url), file=sys.stderr - ) - - self.rev = rev or branch or "main" - self.origin_repo = repo_url - self.manifest = manifest - self.depth = depth if depth else GitFetcher.DEFAULT_DEPTH - self.branch = branch - - def _update(self) -> ChangeStatus: - current_hash = ( - subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=self.repo_dir) - .strip() - .decode("utf-8") - ) - target_hash = ( - subprocess.check_output(["git", "rev-parse", self.rev], cwd=self.repo_dir) - .strip() - .decode("utf-8") - ) - if target_hash == current_hash: - # It's up to date, so there are no changes. This doesn't detect eg: - # if origin/main moved and rev='main', but that's ok for our purposes; - # we should be using explicit hashes or eg: a stable branch for the cases - # that we care about, and it isn't unreasonable to require that the user - # explicitly perform a clean build if those have moved. For the most - # part we prefer that folks build using a release tarball from github - # rather than use the git protocol, as it is generally a bit quicker - # to fetch and easier to hash and verify tarball downloads. - return ChangeStatus() - - print("Updating %s -> %s" % (self.repo_dir, self.rev)) - run_cmd(["git", "fetch", "origin", self.rev], cwd=self.repo_dir) - run_cmd(["git", "checkout", self.rev], cwd=self.repo_dir) - run_cmd(["git", "submodule", "update", "--init"], cwd=self.repo_dir) - - return ChangeStatus(True) - - def update(self) -> ChangeStatus: - if os.path.exists(self.repo_dir): - return self._update() - self._clone() - return ChangeStatus(True) - - def _clone(self) -> None: - print("Cloning %s..." % self.origin_repo) - # The basename/dirname stuff allows us to dance around issues where - # eg: this python process is native win32, but the git.exe is cygwin - # or msys and doesn't like the absolute windows path that we'd otherwise - # pass to it. Careful use of cwd helps avoid headaches with cygpath. - cmd = [ - "git", - "clone", - "--depth=" + str(self.depth), - ] - if self.branch: - cmd.append("--branch=" + self.branch) - cmd += [ - "--", - self.origin_repo, - os.path.basename(self.repo_dir), - ] - run_cmd(cmd, cwd=os.path.dirname(self.repo_dir)) - self._update() - - def clean(self) -> None: - if os.path.exists(self.repo_dir): - run_cmd(["git", "clean", "-fxd"], cwd=self.repo_dir) - - def hash(self): - return self.rev - - def get_src_dir(self): - return self.repo_dir - - -def does_file_need_update(src_name, src_st, dest_name): - try: - target_st = os.lstat(dest_name) - except OSError as exc: - if exc.errno != errno.ENOENT: - raise - return True - - if src_st.st_size != target_st.st_size: - return True - - if stat.S_IFMT(src_st.st_mode) != stat.S_IFMT(target_st.st_mode): - return True - if stat.S_ISLNK(src_st.st_mode): - return os.readlink(src_name) != os.readlink(dest_name) - if not stat.S_ISREG(src_st.st_mode): - return True - - # They might have the same content; compare. - with open(src_name, "rb") as sf, open(dest_name, "rb") as df: - chunk_size = 8192 - while True: - src_data = sf.read(chunk_size) - dest_data = df.read(chunk_size) - if src_data != dest_data: - return True - if len(src_data) < chunk_size: - # EOF - break - return False - - -def copy_if_different(src_name, dest_name) -> bool: - """Copy src_name -> dest_name, but only touch dest_name - if src_name is different from dest_name, making this a - more build system friendly way to copy.""" - src_st = os.lstat(src_name) - if not does_file_need_update(src_name, src_st, dest_name): - return False - - dest_parent = os.path.dirname(dest_name) - if not os.path.exists(dest_parent): - os.makedirs(dest_parent) - if stat.S_ISLNK(src_st.st_mode): - try: - os.unlink(dest_name) - except OSError as exc: - if exc.errno != errno.ENOENT: - raise - target = os.readlink(src_name) - os.symlink(target, dest_name) - else: - shutil.copy2(src_name, dest_name) - - return True - - -def list_files_under_dir_newer_than_timestamp(dir_to_scan, ts): - for root, _dirs, files in os.walk(dir_to_scan): - for src_file in files: - full_name = os.path.join(root, src_file) - st = os.lstat(full_name) - if st.st_mtime > ts: - yield full_name - - -class ShipitPathMap(object): - def __init__(self) -> None: - self.roots = [] - self.mapping = [] - self.exclusion = [] - - def add_mapping(self, fbsource_dir, target_dir) -> None: - """Add a posix path or pattern. We cannot normpath the input - here because that would change the paths from posix to windows - form and break the logic throughout this class.""" - self.roots.append(fbsource_dir) - self.mapping.append((fbsource_dir, target_dir)) - - def add_exclusion(self, pattern) -> None: - self.exclusion.append(re.compile(pattern)) - - def _minimize_roots(self) -> None: - """compute the de-duplicated set of roots within fbsource. - We take the shortest common directory prefix to make this - determination""" - self.roots.sort(key=len) - minimized = [] - - for r in self.roots: - add_this_entry = True - for existing in minimized: - if r.startswith(existing + "/"): - add_this_entry = False - break - if add_this_entry: - minimized.append(r) - - self.roots = minimized - - def _sort_mapping(self) -> None: - self.mapping.sort(reverse=True, key=lambda x: len(x[0])) - - def _map_name(self, norm_name, dest_root): - if norm_name.endswith(".pyc") or norm_name.endswith(".swp"): - # Ignore some incidental garbage while iterating - return None - - for excl in self.exclusion: - if excl.match(norm_name): - return None - - for src_name, dest_name in self.mapping: - if norm_name == src_name or norm_name.startswith(src_name + "/"): - rel_name = os.path.relpath(norm_name, src_name) - # We can have "." as a component of some paths, depending - # on the contents of the shipit transformation section. - # normpath doesn't always remove `.` as the final component - # of the path, which be problematic when we later mkdir - # the dirname of the path that we return. Take care to avoid - # returning a path with a `.` in it. - rel_name = os.path.normpath(rel_name) - if dest_name == ".": - return os.path.normpath(os.path.join(dest_root, rel_name)) - dest_name = os.path.normpath(dest_name) - return os.path.normpath(os.path.join(dest_root, dest_name, rel_name)) - - raise Exception("%s did not match any rules" % norm_name) - - def mirror(self, fbsource_root, dest_root) -> ChangeStatus: - self._minimize_roots() - self._sort_mapping() - - change_status = ChangeStatus() - - # Record the full set of files that should be in the tree - full_file_list = set() - - if sys.platform == "win32": - # Let's not assume st_dev has a consistent value on Windows. - def st_dev(path): - return 1 - - else: - - def st_dev(path): - return os.lstat(path).st_dev - - for fbsource_subdir in self.roots: - dir_to_mirror = os.path.join(fbsource_root, fbsource_subdir) - root_dev = st_dev(dir_to_mirror) - prefetch_dir_if_eden(dir_to_mirror) - if not os.path.exists(dir_to_mirror): - raise Exception( - "%s doesn't exist; check your sparse profile!" % dir_to_mirror - ) - update_count = 0 - for root, dirs, files in os.walk(dir_to_mirror): - dirs[:] = [d for d in dirs if root_dev == st_dev(os.path.join(root, d))] - - for src_file in files: - full_name = os.path.join(root, src_file) - rel_name = os.path.relpath(full_name, fbsource_root) - norm_name = rel_name.replace("\\", "/") - - target_name = self._map_name(norm_name, dest_root) - if target_name: - full_file_list.add(target_name) - if copy_if_different(full_name, target_name): - change_status.record_change(target_name) - if update_count < 10: - print("Updated %s -> %s" % (full_name, target_name)) - elif update_count == 10: - print("...") - update_count += 1 - if update_count: - print("Updated %s for %s" % (update_count, fbsource_subdir)) - - # Compare the list of previously shipped files; if a file is - # in the old list but not the new list then it has been - # removed from the source and should be removed from the - # destination. - # Why don't we simply create this list by walking dest_root? - # Some builds currently have to be in-source builds and - # may legitimately need to keep some state in the source tree :-/ - installed_name = os.path.join(dest_root, ".shipit_shipped") - if os.path.exists(installed_name): - with open(installed_name, "rb") as f: - for name in f.read().decode("utf-8").splitlines(): - name = name.strip() - if name not in full_file_list: - print("Remove %s" % name) - os.unlink(name) - change_status.record_change(name) - - with open(installed_name, "wb") as f: - for name in sorted(list(full_file_list)): - f.write(("%s\n" % name).encode("utf-8")) - - return change_status - - -class FbsourceRepoData(NamedTuple): - hash: str - date: str - - -FBSOURCE_REPO_DATA: Dict[str, FbsourceRepoData] = {} - - -def get_fbsource_repo_data(build_options) -> FbsourceRepoData: - """Returns the commit metadata for the fbsource repo. - Since we may have multiple first party projects to - hash, and because we don't mutate the repo, we cache - this hash in a global.""" - cached_data = FBSOURCE_REPO_DATA.get(build_options.fbsource_dir) - if cached_data: - return cached_data - - if "GETDEPS_HG_REPO_DATA" in os.environ: - log_data = os.environ["GETDEPS_HG_REPO_DATA"] - else: - cmd = ["hg", "log", "-r.", "-T{node}\n{date|hgdate}"] - env = Env() - env.set("HGPLAIN", "1") - log_data = subprocess.check_output( - cmd, cwd=build_options.fbsource_dir, env=dict(env.items()) - ).decode("ascii") - - (hash, datestr) = log_data.split("\n") - - # datestr is like "seconds fractionalseconds" - # We want "20200324.113140" - (unixtime, _fractional) = datestr.split(" ") - date = datetime.fromtimestamp(int(unixtime)).strftime("%Y%m%d.%H%M%S") - cached_data = FbsourceRepoData(hash=hash, date=date) - - FBSOURCE_REPO_DATA[build_options.fbsource_dir] = cached_data - - return cached_data - - -def is_public_commit(build_options) -> bool: # noqa: C901 - """Check if the current commit is public (shipped/will be shipped to remote). - - Works across git, sapling (sl), and hg repositories: - - For hg/sapling: Uses 'phase' command to check if commit is public - - For git: Checks if commit exists in remote branches - - Returns True if public, False if draft/local-only or on error (conservative). - """ - # Use fbsource_dir if available (Meta internal), otherwise fall back to repo_root - repo_dir = build_options.fbsource_dir or build_options.repo_root - if not repo_dir: - # No repository detected, be conservative - return False - - env = Env() - env.set("HGPLAIN", "1") - env_dict = dict(env.items()) - - try: - # Try hg/sapling phase command first (works for both hg and sl) - # Try 'sl' first as it's the preferred tool at Meta - for cmd in [["sl", "phase", "-r", "."], ["hg", "phase", "-r", "."]]: - try: - output = ( - subprocess.check_output( - cmd, cwd=repo_dir, env=env_dict, stderr=subprocess.DEVNULL - ) - .decode("ascii") - .strip() - ) - # Output format: "hash: public" or "hash: draft" - return "public" in output - except (subprocess.CalledProcessError, FileNotFoundError): - continue - - # Try git if hg/sl didn't work - try: - # Detect the default branch for origin remote - default_branch = None - try: - # Get the symbolic ref for origin/HEAD to find default branch - output = ( - subprocess.check_output( - ["git", "symbolic-ref", "refs/remotes/origin/HEAD"], - cwd=repo_dir, - stderr=subprocess.DEVNULL, - ) - .decode("ascii") - .strip() - ) - # Output format: "refs/remotes/origin/main" - if output.startswith("refs/remotes/"): - default_branch = output - except subprocess.CalledProcessError: - # If symbolic-ref fails, fall back to common names - pass - - # Build list of branches to check - branches_to_check = [] - if default_branch: - branches_to_check.append(default_branch) - # Also try common defaults as fallback - branches_to_check.extend(["origin/main", "origin/master"]) - - # Check if HEAD is an ancestor of any of these branches - for branch in branches_to_check: - try: - subprocess.check_output( - ["git", "merge-base", "--is-ancestor", "HEAD", branch], - cwd=repo_dir, - stderr=subprocess.DEVNULL, - ) - # If command succeeds (exit 0), HEAD is an ancestor of the branch - return True - except subprocess.CalledProcessError: - # Not an ancestor of this branch, try next - continue - # HEAD is not in any default branch - return False - except FileNotFoundError: - pass - - # If all VCS commands failed, be conservative and don't upload - return False - - except Exception: - # On any unexpected error, be conservative and don't upload - return False - - -class SimpleShipitTransformerFetcher(Fetcher): - def __init__(self, build_options, manifest, ctx) -> None: - self.build_options = build_options - self.manifest = manifest - self.repo_dir = os.path.join(build_options.scratch_dir, "shipit", manifest.name) - self.ctx = ctx - - def clean(self) -> None: - if os.path.exists(self.repo_dir): - shutil.rmtree(self.repo_dir) - - def update(self) -> ChangeStatus: - mapping = ShipitPathMap() - for src, dest in self.manifest.get_section_as_ordered_pairs( - "shipit.pathmap", self.ctx - ): - mapping.add_mapping(src, dest) - if self.manifest.shipit_fbcode_builder: - mapping.add_mapping( - "fbcode/opensource/fbcode_builder", "build/fbcode_builder" - ) - for pattern in self.manifest.get_section_as_args("shipit.strip", self.ctx): - mapping.add_exclusion(pattern) - - return mapping.mirror(self.build_options.fbsource_dir, self.repo_dir) - - # pyre-fixme[15]: `hash` overrides method defined in `Fetcher` inconsistently. - def hash(self) -> str: - # We return a fixed non-hash string for in-fbsource builds. - # We're relying on the `update` logic to correctly invalidate - # the build in the case that files have changed. - return "fbsource" - - def get_src_dir(self): - return self.repo_dir - - -class SubFetcher(Fetcher): - """Fetcher for a project with subprojects""" - - def __init__(self, base, subs) -> None: - self.base = base - self.subs = subs - - def update(self) -> ChangeStatus: - base = self.base.update() - changed = base.build_changed() or base.sources_changed() - for fetcher, dir in self.subs: - stat = fetcher.update() - if stat.build_changed() or stat.sources_changed(): - changed = True - link = self.base.get_src_dir() + "/" + dir - if not os.path.exists(link): - os.symlink(fetcher.get_src_dir(), link) - return ChangeStatus(changed) - - def clean(self) -> None: - self.base.clean() - for fetcher, _ in self.subs: - fetcher.clean() - - def hash(self) -> None: - hash = self.base.hash() - for fetcher, _ in self.subs: - hash += fetcher.hash() - - def get_src_dir(self): - return self.base.get_src_dir() - - -class ShipitTransformerFetcher(Fetcher): - @classmethod - def _shipit_paths(cls, build_options): - www_path = ["/var/www/scripts/opensource/codesync"] - if build_options.fbsource_dir: - fbcode_path = [ - os.path.join( - build_options.fbsource_dir, - "fbcode/opensource/codesync/codesync-cli/codesync", - ) - ] - else: - fbcode_path = [] - return www_path + fbcode_path - - def __init__(self, build_options, project_name, external_branch) -> None: - self.build_options = build_options - self.project_name = project_name - self.external_branch = external_branch - self.repo_dir = os.path.join(build_options.scratch_dir, "shipit", project_name) - self.shipit = None - for path in ShipitTransformerFetcher._shipit_paths(build_options): - if os.path.exists(path): - self.shipit = path - break - - def update(self) -> ChangeStatus: - if os.path.exists(self.repo_dir): - return ChangeStatus() - self.run_shipit() - return ChangeStatus(True) - - def clean(self) -> None: - if os.path.exists(self.repo_dir): - shutil.rmtree(self.repo_dir) - - @classmethod - def available(cls, build_options): - return any( - os.path.exists(path) - for path in ShipitTransformerFetcher._shipit_paths(build_options) - ) - - def run_shipit(self) -> None: - tmp_path = self.repo_dir + ".new" - try: - if os.path.exists(tmp_path): - shutil.rmtree(tmp_path) - os.makedirs(os.path.dirname(tmp_path), exist_ok=True) - cmd = [ - self.shipit, - "shipit", - "--project=" + self.project_name, - "--create-new-repo", - "--source-repo-dir=" + self.build_options.fbsource_dir, - "--source-branch=.", - "--skip-source-init", - "--skip-source-pull", - "--skip-source-clean", - "--skip-push", - "--destination-use-anonymous-https", - "--create-new-repo-output-path=" + tmp_path, - ] - if self.external_branch: - cmd += [ - f"--external-branch={self.external_branch}", - ] - - # Run shipit - run_cmd(cmd) - - # Remove the .git directory from the repository it generated. - # There is no need to commit this. - repo_git_dir = os.path.join(tmp_path, ".git") - shutil.rmtree(repo_git_dir) - os.rename(tmp_path, self.repo_dir) - except Exception: - # Clean up after a failed extraction - if os.path.exists(tmp_path): - shutil.rmtree(tmp_path) - self.clean() - raise - - # pyre-fixme[15]: `hash` overrides method defined in `Fetcher` inconsistently. - def hash(self) -> str: - # We return a fixed non-hash string for in-fbsource builds. - return "fbsource" - - def get_src_dir(self): - return self.repo_dir - - -def download_url_to_file_with_progress(url: str, file_name) -> None: - print("Download with %s -> %s ..." % (url, file_name)) - - class Progress(object): - last_report = 0 - - def write_update(self, total, amount): - if total == -1: - total = "(Unknown)" - - if sys.stdout.isatty(): - sys.stdout.write("\r downloading %s of %s " % (amount, total)) - else: - # When logging to CI logs, avoid spamming the logs and print - # status every few seconds - now = time.time() - if now - self.last_report > 5: - sys.stdout.write(".. %s of %s " % (amount, total)) - self.last_report = now - sys.stdout.flush() - - def progress_pycurl(self, total, amount, _uploadtotal, _uploadamount): - self.write_update(total, amount) - - progress = Progress() - start = time.time() - try: - if os.environ.get("GETDEPS_USE_WGET") is not None: - procargs = ( - [ - "wget", - ] - + os.environ.get("GETDEPS_WGET_ARGS", "").split() - + [ - "-O", - file_name, - url, - ] - ) - subprocess.run(procargs, capture_output=True) - headers = None - - elif os.environ.get("GETDEPS_USE_LIBCURL") is not None: - import pycurl - - with open(file_name, "wb") as f: - c = pycurl.Curl() - c.setopt(pycurl.URL, url) - c.setopt(pycurl.WRITEDATA, f) - # display progress - c.setopt(pycurl.NOPROGRESS, False) - c.setopt(pycurl.XFERINFOFUNCTION, progress.progress_pycurl) - c.perform() - c.close() - headers = None - else: - try: - req_header = {"Accept": "application/*"} - res = urlopen(Request(url, None, req_header)) - chunk_size = 8192 # urlretrieve uses this value - headers = res.headers - content_length = res.headers.get("Content-Length") - total = int(content_length.strip()) if content_length else -1 - amount = 0 - with open(file_name, "wb") as f: - chunk = res.read(chunk_size) - while chunk: - f.write(chunk) - amount += len(chunk) - progress.write_update(total, amount) - chunk = res.read(chunk_size) - except (OSError, IOError) as exc: # noqa: B014 - # Downloading from within Meta's network needs to use a proxy. - if shutil.which("fwdproxy-config") is None: - print( - "Note: Could not find Meta-specific fallback 'fwdproxy-config'. " - "If you are working externally, you can ignore this message." - ) - raise - - print("Default download failed, retrying with curl and fwdproxy...") - cmd = f"curl -L $(fwdproxy-config curl) -o {shlex.quote(file_name)} {shlex.quote(url)}" - print(f"Running command: {cmd}") - result = subprocess.run(cmd, shell=True, capture_output=True) - if result.returncode != 0: - raise TransientFailure( - f"Failed to download {url} to {file_name}: {exc} (fwdproxy fallback failed: {result.stderr.decode()})" - ) - headers = None - except (OSError, IOError) as exc: # noqa: B014 - raise TransientFailure( - "Failed to download %s to %s: %s" % (url, file_name, str(exc)) - ) - - end = time.time() - sys.stdout.write(" [Complete in %f seconds]\n" % (end - start)) - sys.stdout.flush() - if headers is not None: - print(f"{headers}") - - -class ArchiveFetcher(Fetcher): - def __init__(self, build_options, manifest, url, sha256) -> None: - self.manifest = manifest - self.url = url - self.sha256 = sha256 - self.build_options = build_options - - url = urlparse(self.url) - basename = "%s-%s" % (manifest.name, os.path.basename(url.path)) - self.file_name = os.path.join(build_options.scratch_dir, "downloads", basename) - self.src_dir = os.path.join(build_options.scratch_dir, "extracted", basename) - self.hash_file = self.src_dir + ".hash" - - def _verify_hash(self) -> None: - h = hashlib.sha256() - with open(self.file_name, "rb") as f: - while True: - block = f.read(8192) - if not block: - break - h.update(block) - digest = h.hexdigest() - if digest != self.sha256: - os.unlink(self.file_name) - raise Exception( - "%s: expected sha256 %s but got %s" % (self.url, self.sha256, digest) - ) - - def _download_dir(self): - """returns the download dir, creating it if it doesn't already exist""" - download_dir = os.path.dirname(self.file_name) - if not os.path.exists(download_dir): - os.makedirs(download_dir) - return download_dir - - def _download(self) -> None: - self._download_dir() - max_attempts = 5 - delay = 1 - for attempt in range(max_attempts): - try: - download_url_to_file_with_progress(self.url, self.file_name) - break - except TransientFailure as tf: - if attempt < max_attempts - 1: - delay *= 2 - delay_with_jitter = delay * (1 + random.random() * 0.1) - time.sleep(min(delay_with_jitter, 10)) - else: - print(f"Failed after retries: {tf}") - raise - self._verify_hash() - - def clean(self) -> None: - if os.path.exists(self.src_dir): - shutil.rmtree(self.src_dir) - - def update(self) -> ChangeStatus: - try: - with open(self.hash_file, "r") as f: - saved_hash = f.read().strip() - if saved_hash == self.sha256 and os.path.exists(self.src_dir): - # Everything is up to date - return ChangeStatus() - print( - "saved hash %s doesn't match expected hash %s, re-validating" - % (saved_hash, self.sha256) - ) - os.unlink(self.hash_file) - except EnvironmentError: - pass - - # If we got here we know the contents of src_dir are either missing - # or wrong, so blow away whatever happened to be there first. - if os.path.exists(self.src_dir): - shutil.rmtree(self.src_dir) - - # If we already have a file here, make sure it looks legit before - # proceeding: any errors and we just remove it and re-download - if os.path.exists(self.file_name): - try: - self._verify_hash() - except Exception: - if os.path.exists(self.file_name): - os.unlink(self.file_name) - - if not os.path.exists(self.file_name): - self._download() - self._verify_hash() - - if tarfile.is_tarfile(self.file_name): - opener = tarfile.open - elif zipfile.is_zipfile(self.file_name): - opener = zipfile.ZipFile - else: - raise Exception("don't know how to extract %s" % self.file_name) - os.makedirs(self.src_dir) - print("Extract %s -> %s" % (self.file_name, self.src_dir)) - if is_windows(): - # Ensure that we don't fall over when dealing with long paths - # on windows - src = r"\\?\%s" % os.path.normpath(self.src_dir) - else: - src = self.src_dir - - with opener(self.file_name) as t: - # The `str` here is necessary to ensure that we don't pass a unicode - # object down to tarfile.extractall on python2. When extracting - # the boost tarball it makes some assumptions and tries to convert - # a non-ascii path to ascii and throws. - src = str(src) - t.extractall(src) - - if is_windows(): - subdir = self.manifest.get("build", "subdir") - checkdir = src - if subdir: - checkdir = src + "\\" + subdir - if os.path.exists(checkdir): - children = os.listdir(checkdir) - print(f"Extracted to {checkdir} contents: {children}") - - with open(self.hash_file, "w") as f: - f.write(self.sha256) - - return ChangeStatus(True) - - def hash(self): - return self.sha256 - - def get_src_dir(self): - return self.src_dir - - -def homebrew_package_prefix(package): - cmd = ["brew", "--prefix", package] - try: - proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - except FileNotFoundError: - return - - if proc.returncode == 0: - return proc.stdout.decode("utf-8").rstrip() diff --git a/build/fbcode_builder/getdeps/include_rewriter.py b/build/fbcode_builder/getdeps/include_rewriter.py deleted file mode 100644 index 4024eb3f4..000000000 --- a/build/fbcode_builder/getdeps/include_rewriter.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-unsafe - -""" -Include Path Rewriter for getdeps - -This module provides functionality to rewrite #include statements in C++ files -to handle differences between fbcode and open source project structures. -""" - -import os -import re -from pathlib import Path -from typing import List, Tuple - - -class IncludePathRewriter: - """Rewrites #include paths in C++ source files based on path mappings.""" - - # C++ file extensions to process - CPP_EXTENSIONS = {".cpp", ".cc", ".cxx", ".c", ".h", ".hpp", ".hxx", ".tcc", ".inc"} - - def __init__(self, mappings: List[Tuple[str, str]], verbose: bool = False): - """ - Initialize the rewriter with path mappings. - - Args: - mappings: List of (old_path_prefix, new_path_prefix) tuples - verbose: Enable verbose output - """ - self.mappings = mappings - self.verbose = verbose - - # Compile regex patterns for efficiency - self.patterns = [] - for old_prefix, new_prefix in mappings: - # Match both quoted and angle bracket includes - # Pattern matches: #include "old_prefix/rest" or #include - pattern = re.compile( - r'(#\s*include\s*[<"])(' + re.escape(old_prefix) + r'/[^">]+)([">])', - re.MULTILINE, - ) - self.patterns.append((pattern, old_prefix, new_prefix)) - - def rewrite_file(self, file_path: Path, dry_run: bool = False) -> bool: - """ - Rewrite includes in a single file. - - Args: - file_path: Path to the file to process - dry_run: If True, don't actually modify files - - Returns: - True if file was modified, False otherwise - """ - try: - with open(file_path, "r", encoding="utf-8") as f: - original_content = f.read() - except (IOError, UnicodeDecodeError) as e: - if self.verbose: - print(f"Warning: Could not read {file_path}: {e}") - return False - - modified_content = original_content - changes_made = False - - for pattern, old_prefix, new_prefix in self.patterns: - - def make_replace_func(old_prefix, new_prefix): - def replace_func(match): - nonlocal changes_made - prefix = match.group(1) # #include [<"] - full_path = match.group(2) # full path - suffix = match.group(3) # [">] - - # Replace the old prefix with new prefix - new_path = full_path.replace(old_prefix, new_prefix, 1) - - if self.verbose and not changes_made: - print(f" {full_path} -> {new_path}") - - changes_made = True - return f"{prefix}{new_path}{suffix}" - - return replace_func - - modified_content = pattern.sub( - make_replace_func(old_prefix, new_prefix), modified_content - ) - - if changes_made and not dry_run: - try: - with open(file_path, "w", encoding="utf-8") as f: - f.write(modified_content) - if self.verbose: - print(f"Modified: {file_path}") - except IOError as e: - print(f"Error: Could not write {file_path}: {e}") - return False - elif changes_made and dry_run: - if self.verbose: - print(f"Would modify: {file_path}") - - return changes_made - - def process_directory(self, source_dir: Path, dry_run: bool = False) -> int: - """ - Process all C++ files in a directory recursively. - - Args: - source_dir: Root directory to process - dry_run: If True, don't actually modify files - - Returns: - Number of files modified - """ - if not source_dir.exists(): - if self.verbose: - print(f"Warning: Directory {source_dir} does not exist") - return 0 - - modified_count = 0 - processed_count = 0 - - for root, dirs, files in os.walk(source_dir): - # Skip hidden directories and common build directories - dirs[:] = [ - d - for d in dirs - if not d.startswith(".") - and d not in {"build", "_build", "__pycache__", "CMakeFiles"} - ] - - for file in files: - file_path = Path(root) / file - - # Only process C++ files - if file_path.suffix.lower() not in self.CPP_EXTENSIONS: - continue - - processed_count += 1 - if self.verbose: - print(f"Processing: {file_path}") - - if self.rewrite_file(file_path, dry_run): - modified_count += 1 - - if self.verbose or modified_count > 0: - print(f"Processed {processed_count} files, modified {modified_count} files") - return modified_count - - -def rewrite_includes_from_manifest( - manifest, ctx, source_dir: str, verbose: bool = False -) -> int: - """ - Rewrite includes using mappings from a manifest file. - - Args: - manifest: The manifest object containing shipit.pathmap section - ctx: The manifest context - source_dir: Directory containing source files to process - verbose: Enable verbose output - - Returns: - Number of files modified - """ - mappings = [] - - # Get mappings from the manifest's shipit.pathmap section - for src, dest in manifest.get_section_as_ordered_pairs("shipit.pathmap", ctx): - # Remove fbcode/ or xplat/ prefixes from src since they won't appear in #include statements - if src.startswith("fbcode/"): - src = src[len("fbcode/") :] - elif src.startswith("xplat/"): - src = src[len("xplat/") :] - mappings.append((src, dest)) - - if not mappings: - if verbose: - print("No include path mappings found in manifest") - return 0 - - if verbose: - print("Include path mappings:") - for old_path, new_path in mappings: - print(f" {old_path} -> {new_path}") - - rewriter = IncludePathRewriter(mappings, verbose) - return rewriter.process_directory(Path(source_dir), dry_run=False) diff --git a/build/fbcode_builder/getdeps/load.py b/build/fbcode_builder/getdeps/load.py deleted file mode 100644 index f4c868e9e..000000000 --- a/build/fbcode_builder/getdeps/load.py +++ /dev/null @@ -1,376 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-unsafe - -import base64 -import copy -import hashlib -import os - -from . import fetcher -from .envfuncs import path_search -from .errors import ManifestNotFound -from .manifest import ManifestParser - - -class Loader(object): - """The loader allows our tests to patch the load operation""" - - def _list_manifests(self, build_opts): - """Returns a generator that iterates all the available manifests""" - for path, _, files in os.walk(build_opts.manifests_dir): - for name in files: - # skip hidden files - if name.startswith("."): - continue - - yield os.path.join(path, name) - - def _load_manifest(self, path): - return ManifestParser(path) - - def load_project(self, build_opts, project_name): - if "/" in project_name or "\\" in project_name: - # Assume this is a path already - return ManifestParser(project_name) - - for manifest in self._list_manifests(build_opts): - if os.path.basename(manifest) == project_name: - return ManifestParser(manifest) - - raise ManifestNotFound(project_name) - - def load_all(self, build_opts): - manifests_by_name = {} - - for manifest in self._list_manifests(build_opts): - m = self._load_manifest(manifest) - - if m.name in manifests_by_name: - raise Exception("found duplicate manifest '%s'" % m.name) - - manifests_by_name[m.name] = m - - return manifests_by_name - - -class ResourceLoader(Loader): - def __init__(self, namespace, manifests_dir) -> None: - self.namespace = namespace - self.manifests_dir = manifests_dir - - def _list_manifests(self, _build_opts): - import pkg_resources - - dirs = [self.manifests_dir] - - while dirs: - current = dirs.pop(0) - for name in pkg_resources.resource_listdir(self.namespace, current): - path = "%s/%s" % (current, name) - - if pkg_resources.resource_isdir(self.namespace, path): - dirs.append(path) - else: - yield "%s/%s" % (current, name) - - def _find_manifest(self, project_name): - for name in self._list_manifests(): - if name.endswith("/%s" % project_name): - return name - - raise ManifestNotFound(project_name) - - def _load_manifest(self, path: str): - import pkg_resources - - contents = pkg_resources.resource_string(self.namespace, path).decode("utf8") - return ManifestParser(file_name=path, fp=contents) - - def load_project(self, build_opts, project_name): - project_name = self._find_manifest(project_name) - return self._load_resource_manifest(project_name) - - -LOADER = Loader() - - -def patch_loader(namespace, manifests_dir: str = "manifests") -> None: - global LOADER - LOADER = ResourceLoader(namespace, manifests_dir) - - -def load_project(build_opts, project_name): - """given the name of a project or a path to a manifest file, - load up the ManifestParser instance for it and return it""" - return LOADER.load_project(build_opts, project_name) - - -def load_all_manifests(build_opts): - return LOADER.load_all(build_opts) - - -class ManifestLoader(object): - """ManifestLoader stores information about project manifest relationships for a - given set of (build options + platform) configuration. - - The ManifestLoader class primarily serves as a location to cache project dependency - relationships and project hash values for this build configuration. - """ - - def __init__(self, build_opts, ctx_gen=None) -> None: - self._loader = LOADER - self.build_opts = build_opts - if ctx_gen is None: - self.ctx_gen = self.build_opts.get_context_generator() - else: - self.ctx_gen = ctx_gen - - self.manifests_by_name = {} - self._loaded_all = False - self._project_hashes = {} - self._fetcher_overrides = {} - self._build_dir_overrides = {} - self._install_dir_overrides = {} - self._install_prefix_overrides = {} - - def load_manifest(self, name): - manifest = self.manifests_by_name.get(name) - if manifest is None: - manifest = self._loader.load_project(self.build_opts, name) - self.manifests_by_name[name] = manifest - return manifest - - def load_all_manifests(self): - if not self._loaded_all: - all_manifests_by_name = self._loader.load_all(self.build_opts) - if self.manifests_by_name: - # To help ensure that we only ever have a single manifest object for a - # given project, and that it can't change once we have loaded it, - # only update our mapping for projects that weren't already loaded. - for name, manifest in all_manifests_by_name.items(): - self.manifests_by_name.setdefault(name, manifest) - else: - self.manifests_by_name = all_manifests_by_name - self._loaded_all = True - - return self.manifests_by_name - - def dependencies_of(self, manifest): - """Returns the dependencies of the given project, not including the project itself, in topological order.""" - return [ - dep - for dep in self.manifests_in_dependency_order(manifest) - if dep != manifest - ] - - def manifests_in_dependency_order(self, manifest=None): - """Compute all dependencies of the specified project. Returns a list of the - dependencies plus the project itself, in topologically sorted order. - - Each entry in the returned list only depends on projects that appear before it - in the list. - - If the input manifest is None, the dependencies for all currently loaded - projects will be computed. i.e., if you call load_all_manifests() followed by - manifests_in_dependency_order() this will return a global dependency ordering of - all projects.""" - # The list of deps that have been fully processed - seen = set() - # The list of deps which have yet to be evaluated. This - # can potentially contain duplicates. - if manifest is None: - deps = list(self.manifests_by_name.values()) - else: - assert manifest.name in self.manifests_by_name - deps = [manifest] - # The list of manifests in dependency order - dep_order = [] - system_packages = {} - - while len(deps) > 0: - m = deps.pop(0) - if m.name in seen: - continue - - # Consider its deps, if any. - # We sort them for increased determinism; we'll produce - # a correct order even if they aren't sorted, but we prefer - # to produce the same order regardless of how they are listed - # in the project manifest files. - ctx = self.ctx_gen.get_context(m.name) - dep_list = m.get_dependencies(ctx) - - dep_count = 0 - for dep_name in dep_list: - # If we're not sure whether it is done, queue it up - if dep_name not in seen: - dep = self.manifests_by_name.get(dep_name) - if dep is None: - dep = self._loader.load_project(self.build_opts, dep_name) - self.manifests_by_name[dep.name] = dep - - deps.append(dep) - dep_count += 1 - - if dep_count > 0: - # If we queued anything, re-queue this item, as it depends - # those new item(s) and their transitive deps. - deps.append(m) - continue - - # Its deps are done, so we can emit it - seen.add(m.name) - # Capture system packages as we may need to set PATHs to then later - if ( - self.build_opts.allow_system_packages - and self.build_opts.host_type.get_package_manager() - ): - packages = m.get_required_system_packages(ctx) - for pkg_type, v in packages.items(): - merged = system_packages.get(pkg_type, []) - if v not in merged: - merged += v - system_packages[pkg_type] = merged - # A manifest depends on all system packages in it dependencies as well - m.resolved_system_packages = copy.copy(system_packages) - dep_order.append(m) - - return dep_order - - def set_project_src_dir(self, project_name, path) -> None: - self._fetcher_overrides[project_name] = fetcher.LocalDirFetcher(path) - - def set_project_build_dir(self, project_name, path) -> None: - self._build_dir_overrides[project_name] = path - - def set_project_install_dir(self, project_name, path) -> None: - self._install_dir_overrides[project_name] = path - - def set_project_install_prefix(self, project_name, path) -> None: - self._install_prefix_overrides[project_name] = path - - def create_fetcher(self, manifest): - override = self._fetcher_overrides.get(manifest.name) - if override is not None: - return override - - ctx = self.ctx_gen.get_context(manifest.name) - return manifest.create_fetcher(self.build_opts, self, ctx) - - def get_project_hash(self, manifest): - h = self._project_hashes.get(manifest.name) - if h is None: - h = self._compute_project_hash(manifest) - self._project_hashes[manifest.name] = h - return h - - def _compute_project_hash(self, manifest) -> str: - """This recursive function computes a hash for a given manifest. - The hash takes into account some environmental factors on the - host machine and includes the hashes of its dependencies. - No caching of the computation is performed, which is theoretically - wasteful but the computation is fast enough that it is not required - to cache across multiple invocations.""" - ctx = self.ctx_gen.get_context(manifest.name) - - hasher = hashlib.sha256() - # Some environmental and configuration things matter - env = {} - env["install_dir"] = self.build_opts.install_dir - env["scratch_dir"] = self.build_opts.scratch_dir - env["vcvars_path"] = self.build_opts.vcvars_path - env["os"] = self.build_opts.host_type.ostype - env["distro"] = self.build_opts.host_type.distro - env["distro_vers"] = self.build_opts.host_type.distrovers - env["shared_libs"] = str(self.build_opts.shared_libs) - for name in [ - "CXXFLAGS", - "CPPFLAGS", - "LDFLAGS", - "CXX", - "CC", - "GETDEPS_CMAKE_DEFINES", - ]: - env[name] = os.environ.get(name) - for tool in ["cc", "c++", "gcc", "g++", "clang", "clang++"]: - env["tool-%s" % tool] = path_search(os.environ, tool) - for name in manifest.get_section_as_args("depends.environment", ctx): - env[name] = os.environ.get(name) - - fetcher = self.create_fetcher(manifest) - env["fetcher.hash"] = fetcher.hash() - - for name in sorted(env.keys()): - hasher.update(name.encode("utf-8")) - value = env.get(name) - if value is not None: - try: - hasher.update(value.encode("utf-8")) - except AttributeError as exc: - raise AttributeError("name=%r, value=%r: %s" % (name, value, exc)) - - manifest.update_hash(hasher, ctx) - - # If a patchfile is specified, include its contents in the hash - patchfile = manifest.get("build", "patchfile", ctx=ctx) - if patchfile: - patchfile_path = os.path.join( - self.build_opts.fbcode_builder_dir, "patches", patchfile - ) - if os.path.exists(patchfile_path): - with open(patchfile_path, "rb") as f: - hasher.update(f.read()) - - dep_list = manifest.get_dependencies(ctx) - for dep in dep_list: - dep_manifest = self.load_manifest(dep) - dep_hash = self.get_project_hash(dep_manifest) - hasher.update(dep_hash.encode("utf-8")) - - # Use base64 to represent the hash, rather than the simple hex digest, - # so that the string is shorter. Use the URL-safe encoding so that - # the hash can also be safely used as a filename component. - h = base64.urlsafe_b64encode(hasher.digest()).decode("ascii") - # ... and because cmd.exe is troublesome with `=` signs, nerf those. - # They tend to be padding characters at the end anyway, so we can - # safely discard them. - h = h.replace("=", "") - - return h - - def _get_project_dir_name(self, manifest): - if manifest.is_first_party_project(): - return manifest.name - else: - project_hash = self.get_project_hash(manifest) - return "%s-%s" % (manifest.name, project_hash) - - def get_project_install_dir(self, manifest): - override = self._install_dir_overrides.get(manifest.name) - if override: - return override - - project_dir_name = self._get_project_dir_name(manifest) - return os.path.join(self.build_opts.install_dir, project_dir_name) - - def get_project_build_dir(self, manifest): - override = self._build_dir_overrides.get(manifest.name) - if override: - return override - - project_dir_name = self._get_project_dir_name(manifest) - return os.path.join(self.build_opts.scratch_dir, "build", project_dir_name) - - def get_project_install_prefix(self, manifest): - return self._install_prefix_overrides.get(manifest.name) - - def get_project_install_dir_respecting_install_prefix(self, manifest): - inst_dir = self.get_project_install_dir(manifest) - prefix = self.get_project_install_prefix(manifest) - if prefix: - return inst_dir + prefix - return inst_dir diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py deleted file mode 100644 index c68def36d..000000000 --- a/build/fbcode_builder/getdeps/manifest.py +++ /dev/null @@ -1,840 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-unsafe - -import configparser -import io -import os -import sys -from typing import List - -from .builder import ( - AutoconfBuilder, - Boost, - CMakeBootStrapBuilder, - CMakeBuilder, - Iproute2Builder, - MakeBuilder, - MesonBuilder, - NinjaBootstrap, - NopBuilder, - OpenSSLBuilder, - SetupPyBuilder, - SqliteBuilder, -) -from .cargo import CargoBuilder -from .expr import parse_expr -from .fetcher import ( - ArchiveFetcher, - GitFetcher, - PreinstalledNopFetcher, - ShipitTransformerFetcher, - SimpleShipitTransformerFetcher, - SubFetcher, - SystemPackageFetcher, -) -from .py_wheel_builder import PythonWheelBuilder - -REQUIRED = "REQUIRED" -OPTIONAL = "OPTIONAL" - -SCHEMA = { - "manifest": { - "optional_section": False, - "fields": { - "name": REQUIRED, - "fbsource_path": OPTIONAL, - "shipit_project": OPTIONAL, - "shipit_fbcode_builder": OPTIONAL, - "use_shipit": OPTIONAL, - "shipit_external_branch": OPTIONAL, - }, - }, - "dependencies": {"optional_section": True, "allow_values": False}, - "depends.environment": {"optional_section": True}, - "git": { - "optional_section": True, - "fields": { - "repo_url": REQUIRED, - "rev": OPTIONAL, - "depth": OPTIONAL, - "branch": OPTIONAL, - }, - }, - "download": { - "optional_section": True, - "fields": {"url": REQUIRED, "sha256": REQUIRED}, - }, - "build": { - "optional_section": True, - "fields": { - "builder": REQUIRED, - "subdir": OPTIONAL, - "make_binary": OPTIONAL, - "build_in_src_dir": OPTIONAL, - "only_install": OPTIONAL, - "job_weight_mib": OPTIONAL, - "patchfile": OPTIONAL, - "patchfile_opts": OPTIONAL, - "rewrite_includes": OPTIONAL, - }, - }, - "msbuild": {"optional_section": True, "fields": {"project": REQUIRED}}, - "cargo": { - "optional_section": True, - "fields": { - "build_doc": OPTIONAL, - "workspace_dir": OPTIONAL, - "manifests_to_build": OPTIONAL, - # Where to write cargo config (defaults to build_dir/.cargo/config.toml) - "cargo_config_file": OPTIONAL, - }, - }, - "github.actions": { - "optional_section": True, - "fields": { - "run_tests": OPTIONAL, - "required_locales": OPTIONAL, - "rust_version": OPTIONAL, - "build_type": OPTIONAL, - }, - }, - "crate.pathmap": {"optional_section": True}, - "cmake.defines": {"optional_section": True}, - "autoconf.args": {"optional_section": True}, - "autoconf.envcmd.LDFLAGS": {"optional_section": True}, - "rpms": {"optional_section": True}, - "debs": {"optional_section": True}, - "homebrew": {"optional_section": True}, - "pps": {"optional_section": True}, - "preinstalled.env": {"optional_section": True}, - "bootstrap.args": {"optional_section": True}, - "b2.args": {"optional_section": True}, - "make.build_args": {"optional_section": True}, - "make.install_args": {"optional_section": True}, - "make.test_args": {"optional_section": True}, - "meson.setup_args": {"optional_section": True}, - "header-only": {"optional_section": True, "fields": {"includedir": REQUIRED}}, - "shipit.pathmap": {"optional_section": True}, - "shipit.strip": {"optional_section": True}, - "install.files": {"optional_section": True}, - "subprojects": {"optional_section": True}, - # fb-only - "sandcastle": {"optional_section": True, "fields": {"run_tests": OPTIONAL}}, - "setup-py.test": {"optional_section": True, "fields": {"python_script": REQUIRED}}, - "setup-py.env": {"optional_section": True}, -} - -# These sections are allowed to vary for different platforms -# using the expression syntax to enable/disable sections -ALLOWED_EXPR_SECTIONS = [ - "autoconf.args", - "autoconf.envcmd.LDFLAGS", - "build", - "cmake.defines", - "dependencies", - "make.build_args", - "make.install_args", - "bootstrap.args", - "b2.args", - "download", - "git", - "install.files", - "rpms", - "debs", - "shipit.pathmap", - "shipit.strip", - "homebrew", - "github.actions", - "pps", -] - - -def parse_conditional_section_name(name, section_def): - expr = name[len(section_def) + 1 :] - return parse_expr(expr, ManifestContext.ALLOWED_VARIABLES) - - -def validate_allowed_fields(file_name, section, config, allowed_fields): - for field in config.options(section): - if not allowed_fields.get(field): - raise Exception( - ("manifest file %s section '%s' contains " "unknown field '%s'") - % (file_name, section, field) - ) - - for field in allowed_fields: - if allowed_fields[field] == REQUIRED and not config.has_option(section, field): - raise Exception( - ("manifest file %s section '%s' is missing " "required field '%s'") - % (file_name, section, field) - ) - - -def validate_allow_values(file_name, section, config): - for field in config.options(section): - value = config.get(section, field) - if value is not None: - raise Exception( - ( - "manifest file %s section '%s' has '%s = %s' but " - "this section doesn't allow specifying values " - "for its entries" - ) - % (file_name, section, field, value) - ) - - -def validate_section(file_name, section, config): - section_def = SCHEMA.get(section) - if not section_def: - for name in ALLOWED_EXPR_SECTIONS: - if section.startswith(name + "."): - # Verify that the conditional parses, but discard it - try: - parse_conditional_section_name(section, name) - except Exception as exc: - raise Exception( - ("manifest file %s section '%s' has invalid " "conditional: %s") - % (file_name, section, str(exc)) - ) - section_def = SCHEMA.get(name) - canonical_section_name = name - break - if not section_def: - raise Exception( - "manifest file %s contains unknown section '%s'" % (file_name, section) - ) - else: - canonical_section_name = section - - allowed_fields = section_def.get("fields") - if allowed_fields: - validate_allowed_fields(file_name, section, config, allowed_fields) - elif not section_def.get("allow_values", True): - validate_allow_values(file_name, section, config) - return canonical_section_name - - -class ManifestParser(object): - def __init__(self, file_name, fp=None): - # allow_no_value enables listing parameters in the - # autoconf.args section one per line - config = configparser.RawConfigParser(allow_no_value=True) - config.optionxform = str # make it case sensitive - if fp is None: - with open(file_name, "r") as fp: - config.read_file(fp) - elif isinstance(fp, type("")): - # For testing purposes, parse from a string (str - # or unicode) - config.read_file(io.StringIO(fp)) - else: - config.read_file(fp) - - # validate against the schema - seen_sections = set() - - for section in config.sections(): - seen_sections.add(validate_section(file_name, section, config)) - - for section in SCHEMA.keys(): - section_def = SCHEMA[section] - if ( - not section_def.get("optional_section", False) - and section not in seen_sections - ): - raise Exception( - "manifest file %s is missing required section %s" - % (file_name, section) - ) - - self._config = config - self.name = config.get("manifest", "name") - self.fbsource_path = self.get("manifest", "fbsource_path") - self.shipit_project = self.get("manifest", "shipit_project") - self.shipit_fbcode_builder = self.get("manifest", "shipit_fbcode_builder") - self.resolved_system_packages = {} - - if self.name != os.path.basename(file_name): - raise Exception( - "filename of the manifest '%s' does not match the manifest name '%s'" - % (file_name, self.name) - ) - - if "." in self.name: - raise Exception( - f"manifest name ({self.name}) must not contain the '.' character (it is incompatible with github actions)" - ) - - def get(self, section, key, defval=None, ctx=None): - ctx = ctx or {} - - for s in self._config.sections(): - if s == section: - if self._config.has_option(s, key): - return self._config.get(s, key) - return defval - - if s.startswith(section + "."): - expr = parse_conditional_section_name(s, section) - if not expr.eval(ctx): - continue - - if self._config.has_option(s, key): - return self._config.get(s, key) - - return defval - - def get_dependencies(self, ctx): - dep_list = list(self.get_section_as_dict("dependencies", ctx).keys()) - dep_list.sort() - builder = self.get("build", "builder", ctx=ctx) - if builder in ("cmake", "python-wheel"): - dep_list.insert(0, "cmake") - elif builder == "autoconf" and self.name not in ( - "autoconf", - "libtool", - "automake", - ): - # they need libtool and its deps (automake, autoconf) so add - # those as deps (but obviously not if we're building those - # projects themselves) - dep_list.insert(0, "libtool") - - return dep_list - - def get_section_as_args(self, section, ctx=None) -> List[str]: - """Intended for use with the make.[build_args/install_args] and - autoconf.args sections, this method collects the entries and returns an - array of strings. - If the manifest contains conditional sections, ctx is used to - evaluate the condition and merge in the values. - """ - args = [] - ctx = ctx or {} - - for s in self._config.sections(): - if s != section: - if not s.startswith(section + "."): - continue - expr = parse_conditional_section_name(s, section) - if not expr.eval(ctx): - continue - for field in self._config.options(s): - value = self._config.get(s, field) - if value is None: - args.append(field) - else: - args.append("%s=%s" % (field, value)) - return args - - def get_section_as_ordered_pairs(self, section, ctx=None): - """Used for eg: shipit.pathmap which has strong - ordering requirements""" - res = [] - ctx = ctx or {} - - for s in self._config.sections(): - if s != section: - if not s.startswith(section + "."): - continue - expr = parse_conditional_section_name(s, section) - if not expr.eval(ctx): - continue - - for key in self._config.options(s): - value = self._config.get(s, key) - res.append((key, value)) - return res - - def get_section_as_dict(self, section, ctx): - d = {} - - for s in self._config.sections(): - if s != section: - if not s.startswith(section + "."): - continue - expr = parse_conditional_section_name(s, section) - if not expr.eval(ctx): - continue - for field in self._config.options(s): - value = self._config.get(s, field) - d[field] = value - return d - - def update_hash(self, hasher, ctx): - """Compute a hash over the configuration for the given - context. The goal is for the hash to change if the config - for that context changes, but not if a change is made to - the config only for a different platform than that expressed - by ctx. The hash is intended to be used to help invalidate - a future cache for the third party build products. - The hasher argument is a hash object returned from hashlib.""" - for section in sorted(SCHEMA.keys()): - hasher.update(section.encode("utf-8")) - - # Note: at the time of writing, nothing in the implementation - # relies on keys in any config section being ordered. - # In theory we could have conflicting flags in different - # config sections and later flags override earlier flags. - # For the purposes of computing a hash we're not super - # concerned about this: manifest changes should be rare - # enough and we'd rather that this trigger an invalidation - # than strive for a cache hit at this time. - pairs = self.get_section_as_ordered_pairs(section, ctx) - pairs.sort(key=lambda pair: pair[0]) - for key, value in pairs: - hasher.update(key.encode("utf-8")) - if value is not None: - hasher.update(value.encode("utf-8")) - - def is_first_party_project(self): - """returns true if this is an FB first-party project""" - return self.shipit_project is not None - - def get_required_system_packages(self, ctx): - """Returns dictionary of packager system -> list of packages""" - return { - "rpm": self.get_section_as_args("rpms", ctx), - "deb": self.get_section_as_args("debs", ctx), - "homebrew": self.get_section_as_args("homebrew", ctx), - "pacman-package": self.get_section_as_args("pps", ctx), - } - - def _is_satisfied_by_preinstalled_environment(self, ctx): - envs = self.get_section_as_args("preinstalled.env", ctx) - if not envs: - return False - for key in envs: - val = os.environ.get(key, None) - print( - f"Testing ENV[{key}]: {repr(val)}", - file=sys.stderr, - ) - if val is None: - return False - if len(val) == 0: - return False - - return True - - def get_repo_url(self, ctx): - return self.get("git", "repo_url", ctx=ctx) - - def _create_fetcher(self, build_options, ctx): - real_shipit_available = ShipitTransformerFetcher.available(build_options) - use_real_shipit = real_shipit_available and ( - build_options.use_shipit - or self.get("manifest", "use_shipit", defval="false", ctx=ctx) == "true" - ) - if ( - not use_real_shipit - and self.fbsource_path - and build_options.fbsource_dir - and self.shipit_project - ): - return SimpleShipitTransformerFetcher(build_options, self, ctx) - - if ( - self.fbsource_path - and build_options.fbsource_dir - and self.shipit_project - and real_shipit_available - ): - # We can use the code from fbsource - return ShipitTransformerFetcher( - build_options, - self.shipit_project, - self.get("manifest", "shipit_external_branch"), - ) - - # If both of these are None, the package can only be coming from - # preinstalled toolchain or system packages - repo_url = self.get_repo_url(ctx) - url = self.get("download", "url", ctx=ctx) - - # Can we satisfy this dep with system packages? - if (repo_url is None and url is None) or build_options.allow_system_packages: - if self._is_satisfied_by_preinstalled_environment(ctx): - return PreinstalledNopFetcher() - - if build_options.host_type.get_package_manager(): - packages = self.get_required_system_packages(ctx) - package_fetcher = SystemPackageFetcher(build_options, packages) - if package_fetcher.packages_are_installed(): - return package_fetcher - - if repo_url: - rev = self.get("git", "rev") - depth = self.get("git", "depth") - branch = self.get("git", "branch") - return GitFetcher(build_options, self, repo_url, rev, depth, branch) - - if url: - # We need to defer this import until now to avoid triggering - # a cycle when the facebook/__init__.py is loaded. - try: - from .facebook.lfs import LFSCachingArchiveFetcher - - return LFSCachingArchiveFetcher( - build_options, self, url, self.get("download", "sha256", ctx=ctx) - ) - except ImportError: - # This FB internal module isn't shippped to github, - # so just use its base class - return ArchiveFetcher( - build_options, self, url, self.get("download", "sha256", ctx=ctx) - ) - - raise KeyError( - f"project {self.name} has no fetcher configuration or system packages matching {ctx} - have you run `getdeps.py install-system-deps --recursive`?" - ) - - def create_fetcher(self, build_options, loader, ctx): - fetcher = self._create_fetcher(build_options, ctx) - subprojects = self.get_section_as_ordered_pairs("subprojects", ctx) - if subprojects: - subs = [] - for project, subdir in subprojects: - submanifest = loader.load_manifest(project) - subfetcher = submanifest.create_fetcher(build_options, loader, ctx) - subs.append((subfetcher, subdir)) - return SubFetcher(fetcher, subs) - else: - return fetcher - - def get_builder_name(self, ctx): - builder = self.get("build", "builder", ctx=ctx) - if not builder: - raise Exception("project %s has no builder for %r" % (self.name, ctx)) - return builder - - def create_builder( # noqa:C901 - self, - build_options, - src_dir, - build_dir, - inst_dir, - ctx, - loader, - dep_manifests, - final_install_prefix=None, - extra_cmake_defines=None, - cmake_targets=None, - extra_b2_args=None, - ): - builder = self.get_builder_name(ctx) - build_in_src_dir = self.get("build", "build_in_src_dir", "false", ctx=ctx) - if build_in_src_dir == "true": - # Some scripts don't work when they are configured and build in - # a different directory than source (or when the build directory - # is not a subdir of source). - build_dir = src_dir - subdir = self.get("build", "subdir", None, ctx=ctx) - if subdir is not None: - build_dir = os.path.join(build_dir, subdir) - print("build_dir is %s" % build_dir) # just to quiet lint - - if builder == "make" or builder == "cmakebootstrap": - build_args = self.get_section_as_args("make.build_args", ctx) - install_args = self.get_section_as_args("make.install_args", ctx) - test_args = self.get_section_as_args("make.test_args", ctx) - if builder == "cmakebootstrap": - return CMakeBootStrapBuilder( - loader, - dep_manifests, - build_options, - ctx, - self, - src_dir, - None, - inst_dir, - build_args, - install_args, - test_args, - ) - else: - return MakeBuilder( - loader, - dep_manifests, - build_options, - ctx, - self, - src_dir, - None, - inst_dir, - build_args, - install_args, - test_args, - ) - - if builder == "autoconf": - args = self.get_section_as_args("autoconf.args", ctx) - conf_env_args = {} - ldflags_cmd = self.get_section_as_args("autoconf.envcmd.LDFLAGS", ctx) - if ldflags_cmd: - conf_env_args["LDFLAGS"] = ldflags_cmd - return AutoconfBuilder( - loader, - dep_manifests, - build_options, - ctx, - self, - src_dir, - build_dir, - inst_dir, - args, - conf_env_args, - ) - - if builder == "boost": - args = self.get_section_as_args("b2.args", ctx) - if extra_b2_args is not None: - args += extra_b2_args - return Boost( - loader, - dep_manifests, - build_options, - ctx, - self, - src_dir, - build_dir, - inst_dir, - args, - ) - - if builder == "cmake": - defines = self.get_section_as_dict("cmake.defines", ctx) - return CMakeBuilder( - loader, - dep_manifests, - build_options, - ctx, - self, - src_dir, - build_dir, - inst_dir, - defines, - final_install_prefix, - extra_cmake_defines, - cmake_targets, - ) - - if builder == "python-wheel": - return PythonWheelBuilder( - loader, - dep_manifests, - build_options, - ctx, - self, - src_dir, - build_dir, - inst_dir, - ) - - if builder == "sqlite": - return SqliteBuilder( - loader, - dep_manifests, - build_options, - ctx, - self, - src_dir, - build_dir, - inst_dir, - ) - - if builder == "ninja_bootstrap": - return NinjaBootstrap( - loader, - dep_manifests, - build_options, - ctx, - self, - build_dir, - src_dir, - inst_dir, - ) - - if builder == "nop": - return NopBuilder( - loader, dep_manifests, build_options, ctx, self, src_dir, inst_dir - ) - - if builder == "openssl": - return OpenSSLBuilder( - loader, - dep_manifests, - build_options, - ctx, - self, - build_dir, - src_dir, - inst_dir, - ) - - if builder == "iproute2": - return Iproute2Builder( - loader, - dep_manifests, - build_options, - ctx, - self, - src_dir, - build_dir, - inst_dir, - ) - - if builder == "meson": - return MesonBuilder( - loader, - dep_manifests, - build_options, - ctx, - self, - src_dir, - build_dir, - inst_dir, - ) - - if builder == "setup-py": - return SetupPyBuilder( - loader, - dep_manifests, - build_options, - ctx, - self, - src_dir, - build_dir, - inst_dir, - ) - - if builder == "cargo": - return self.create_cargo_builder( - loader, - dep_manifests, - build_options, - ctx, - src_dir, - build_dir, - inst_dir, - ) - - raise KeyError("project %s has no known builder" % (self.name)) - - def create_prepare_builders( - self, - build_options, - ctx, - src_dir, - build_dir, - inst_dir, - loader, - dep_manifests, - ): - """Create builders that have a prepare step run, e.g. to write config files""" - prepare_builders = [] - builder = self.get_builder_name(ctx) - cargo = self.get_section_as_dict("cargo", ctx) - if not builder == "cargo" and cargo: - cargo_builder = self.create_cargo_builder( - loader, - dep_manifests, - build_options, - ctx, - src_dir, - build_dir, - inst_dir, - ) - prepare_builders.append(cargo_builder) - return prepare_builders - - def create_cargo_builder( - self, loader, dep_manifests, build_options, ctx, src_dir, build_dir, inst_dir - ): - build_doc = self.get("cargo", "build_doc", False, ctx) - workspace_dir = self.get("cargo", "workspace_dir", None, ctx) - manifests_to_build = self.get("cargo", "manifests_to_build", None, ctx) - cargo_config_file = self.get("cargo", "cargo_config_file", None, ctx) - return CargoBuilder( - loader, - dep_manifests, - build_options, - ctx, - self, - src_dir, - build_dir, - inst_dir, - build_doc, - workspace_dir, - manifests_to_build, - cargo_config_file, - ) - - -class ManifestContext(object): - """ProjectContext contains a dictionary of values to use when evaluating boolean - expressions in a project manifest. - - This object should be passed as the `ctx` parameter in ManifestParser.get() calls. - """ - - ALLOWED_VARIABLES = { - "os", - "distro", - "distro_vers", - "fb", - "fbsource", - "test", - "shared_libs", - } - - def __init__(self, ctx_dict): - assert set(ctx_dict.keys()) == self.ALLOWED_VARIABLES - self.ctx_dict = ctx_dict - - def get(self, key): - return self.ctx_dict[key] - - def set(self, key, value): - assert key in self.ALLOWED_VARIABLES - self.ctx_dict[key] = value - - def copy(self): - return ManifestContext(dict(self.ctx_dict)) - - def __str__(self): - s = ", ".join( - "%s=%s" % (key, value) for key, value in sorted(self.ctx_dict.items()) - ) - return "{" + s + "}" - - -class ContextGenerator(object): - """ContextGenerator allows creating ManifestContext objects on a per-project basis. - This allows us to evaluate different projects with slightly different contexts. - - For instance, this can be used to only enable tests for some projects.""" - - def __init__(self, default_ctx): - self.default_ctx = ManifestContext(default_ctx) - self.ctx_by_project = {} - - def set_value_for_project(self, project_name, key, value): - project_ctx = self.ctx_by_project.get(project_name) - if project_ctx is None: - project_ctx = self.default_ctx.copy() - self.ctx_by_project[project_name] = project_ctx - project_ctx.set(key, value) - - def set_value_for_all_projects(self, key, value): - self.default_ctx.set(key, value) - for ctx in self.ctx_by_project.values(): - ctx.set(key, value) - - def get_context(self, project_name): - return self.ctx_by_project.get(project_name, self.default_ctx) diff --git a/build/fbcode_builder/getdeps/platform.py b/build/fbcode_builder/getdeps/platform.py deleted file mode 100644 index 7807ed106..000000000 --- a/build/fbcode_builder/getdeps/platform.py +++ /dev/null @@ -1,289 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-unsafe - -import os -import platform -import re -import shlex -import sys -from typing import Optional, Tuple - - -def is_windows() -> bool: - """Returns true if the system we are currently running on - is a Windows system""" - return sys.platform.startswith("win") - - -def get_linux_type() -> Tuple[Optional[str], Optional[str], Optional[str]]: - try: - with open("/etc/os-release") as f: - data = f.read() - except EnvironmentError: - return (None, None, None) - - os_vars = {} - for line in data.splitlines(): - parts = line.split("=", 1) - if len(parts) != 2: - continue - key = parts[0].strip() - value_parts = shlex.split(parts[1].strip()) - if not value_parts: - value = "" - else: - value = value_parts[0] - os_vars[key] = value - - name = os_vars.get("NAME") - if name: - name = name.lower() - name = re.sub("linux", "", name) - name = name.strip().replace(" ", "_") - - version_id = os_vars.get("VERSION_ID") - if version_id: - version_id = version_id.lower() - - return "linux", name, version_id - - -# Ideally we'd use a common library like `psutil` to read system information, -# but getdeps can't take third-party dependencies. - - -def _get_available_ram_linux() -> int: - # TODO: Ideally, this function would inspect the current cgroup for any - # limits, rather than solely relying on system RAM. - - meminfo_path = "/proc/meminfo" - try: - with open(meminfo_path) as f: - for line in f: - try: - key, value = line.split(":", 1) - except ValueError: - continue - suffix = " kB\n" - if key == "MemAvailable" and value.endswith(suffix): - value = value[: -len(suffix)] - try: - return int(value) // 1024 - except ValueError: - continue - except OSError: - print("error opening {}".format(meminfo_path), end="", file=sys.stderr) - else: - print( - "{} had no valid MemAvailable".format(meminfo_path), end="", file=sys.stderr - ) - - guess = 8 - print(", guessing {} GiB".format(guess), file=sys.stderr) - return guess * 1024 - - -def _get_available_ram_macos() -> int: - import ctypes.util - - libc = ctypes.CDLL(ctypes.util.find_library("libc"), use_errno=True) - sysctlbyname = libc.sysctlbyname - sysctlbyname.restype = ctypes.c_int - sysctlbyname.argtypes = [ - ctypes.c_char_p, - ctypes.c_void_p, - ctypes.POINTER(ctypes.c_size_t), - ctypes.c_void_p, - ctypes.c_size_t, - ] - # TODO: There may be some way to approximate an availability - # metric, but just use total RAM for now. - memsize = ctypes.c_int64() - memsizesize = ctypes.c_size_t(8) - res = sysctlbyname( - b"hw.memsize", ctypes.byref(memsize), ctypes.byref(memsizesize), None, 0 - ) - if res != 0: - raise NotImplementedError( - f"failed to retrieve hw.memsize sysctl: {ctypes.get_errno()}" - ) - return memsize.value // (1024 * 1024) - - -def _get_available_ram_windows() -> int: - import ctypes - - DWORD = ctypes.c_uint32 - QWORD = ctypes.c_uint64 - - class MEMORYSTATUSEX(ctypes.Structure): - _fields_ = [ - ("dwLength", DWORD), - ("dwMemoryLoad", DWORD), - ("ullTotalPhys", QWORD), - ("ullAvailPhys", QWORD), - ("ullTotalPageFile", QWORD), - ("ullAvailPageFile", QWORD), - ("ullTotalVirtual", QWORD), - ("ullAvailVirtual", QWORD), - ("ullExtendedVirtual", QWORD), - ] - - ms = MEMORYSTATUSEX() - ms.dwLength = ctypes.sizeof(ms) - # pyre-ignore[16] - res = ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(ms)) - if res == 0: - raise NotImplementedError("error calling GlobalMemoryStatusEx") - - # This is fuzzy, but AvailPhys is too conservative, and AvailTotal is too - # aggressive, so average the two. It's okay for builds to use some swap. - return (ms.ullAvailPhys + ms.ullTotalPhys) // (2 * 1024 * 1024) - - -def _get_available_ram_freebsd() -> int: - import ctypes.util - - libc = ctypes.CDLL(ctypes.util.find_library("libc"), use_errno=True) - sysctlbyname = libc.sysctlbyname - sysctlbyname.restype = ctypes.c_int - sysctlbyname.argtypes = [ - ctypes.c_char_p, - ctypes.c_void_p, - ctypes.POINTER(ctypes.c_size_t), - ctypes.c_void_p, - ctypes.c_size_t, - ] - # hw.usermem is pretty close to what we want. - memsize = ctypes.c_int64() - memsizesize = ctypes.c_size_t(8) - res = sysctlbyname( - b"hw.usermem", ctypes.byref(memsize), ctypes.byref(memsizesize), None, 0 - ) - if res != 0: - raise NotImplementedError( - f"failed to retrieve hw.memsize sysctl: {ctypes.get_errno()}" - ) - return memsize.value // (1024 * 1024) - - -def get_available_ram() -> int: - """ - Returns a platform-appropriate available RAM metric in MiB. - """ - if sys.platform == "linux": - return _get_available_ram_linux() - elif sys.platform == "darwin": - return _get_available_ram_macos() - elif sys.platform == "win32": - return _get_available_ram_windows() - elif sys.platform.startswith("freebsd"): - return _get_available_ram_freebsd() - else: - raise NotImplementedError( - f"platform {sys.platform} does not have an implementation of get_available_ram" - ) - - -def is_current_host_arm() -> bool: - if sys.platform.startswith("darwin"): - # platform.machine() can be fooled by rosetta for python < 3.9.2 - return "ARM64" in os.uname().version - else: - machine = platform.machine().lower() - return "arm" in machine or "aarch" in machine - - -class HostType(object): - def __init__(self, ostype=None, distro=None, distrovers=None) -> None: - # Maybe we should allow callers to indicate whether this machine uses - # an ARM architecture, but we need to change HostType serialization - # and deserialization in that case and hunt down anywhere that is - # persisting that serialized data. - isarm = False - - if ostype is None: - distro = None - distrovers = None - if sys.platform.startswith("linux"): - ostype, distro, distrovers = get_linux_type() - elif sys.platform.startswith("darwin"): - ostype = "darwin" - elif is_windows(): - ostype = "windows" - distrovers = str(sys.getwindowsversion().major) - elif sys.platform.startswith("freebsd"): - ostype = "freebsd" - else: - ostype = sys.platform - - isarm = is_current_host_arm() - - # The operating system type - self.ostype = ostype - # The distribution, if applicable - self.distro = distro - # The OS/distro version if known - self.distrovers = distrovers - # Does the CPU use an ARM architecture? ARM includes Apple Silicon - # Macs as well as other ARM systems that might be running Linux or - # something. - self.isarm = isarm - - def is_windows(self): - return self.ostype == "windows" - - # is_arm is kinda half implemented at the moment. This method is only - # intended to be used when HostType represents information about the - # current machine we are running on. - # When HostType is being used to enumerate platform types (represent - # information about machine types that we may or may not be running on) - # the result could be nonsense (under the current implementation its always - # false.) - def is_arm(self): - return self.isarm - - def is_darwin(self): - return self.ostype == "darwin" - - def is_linux(self): - return self.ostype == "linux" - - def is_freebsd(self): - return self.ostype == "freebsd" - - def as_tuple_string(self) -> str: - return "%s-%s-%s" % ( - self.ostype, - self.distro or "none", - self.distrovers or "none", - ) - - def get_package_manager(self): - if not self.is_linux() and not self.is_darwin(): - return None - if self.is_darwin(): - return "homebrew" - if self.distro in ("fedora", "centos", "centos_stream", "rocky"): - return "rpm" - if self.distro.startswith(("debian", "ubuntu", "pop!_os", "mint")): - return "deb" - if self.distro == "arch": - return "pacman-package" - return None - - @staticmethod - def from_tuple_string(s) -> "HostType": - ostype, distro, distrovers = s.split("-") - return HostType(ostype=ostype, distro=distro, distrovers=distrovers) - - def __eq__(self, b): - return ( - self.ostype == b.ostype - and self.distro == b.distro - and self.distrovers == b.distrovers - ) diff --git a/build/fbcode_builder/getdeps/py_wheel_builder.py b/build/fbcode_builder/getdeps/py_wheel_builder.py deleted file mode 100644 index 536155bd1..000000000 --- a/build/fbcode_builder/getdeps/py_wheel_builder.py +++ /dev/null @@ -1,289 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-unsafe - -import codecs -import collections -import email -import os -import re -import stat -from typing import Dict, List - -from .builder import BuilderBase, CMakeBuilder - - -WheelNameInfo = collections.namedtuple( - "WheelNameInfo", ("distribution", "version", "build", "python", "abi", "platform") -) - -CMAKE_HEADER = """ -cmake_minimum_required(VERSION 3.8) - -project("{manifest_name}" LANGUAGES C) - -set(CMAKE_MODULE_PATH - "{cmake_dir}" - ${{CMAKE_MODULE_PATH}} -) -include(FBPythonBinary) - -set(CMAKE_INSTALL_DIR lib/cmake/{manifest_name} CACHE STRING - "The subdirectory where CMake package config files should be installed") -""" - -CMAKE_FOOTER = """ -install_fb_python_library({lib_name} EXPORT all) -install( - EXPORT all - FILE {manifest_name}-targets.cmake - NAMESPACE {namespace}:: - DESTINATION ${{CMAKE_INSTALL_DIR}} -) - -include(CMakePackageConfigHelpers) -configure_package_config_file( - ${{CMAKE_BINARY_DIR}}/{manifest_name}-config.cmake.in - {manifest_name}-config.cmake - INSTALL_DESTINATION ${{CMAKE_INSTALL_DIR}} - PATH_VARS - CMAKE_INSTALL_DIR -) -install( - FILES ${{CMAKE_CURRENT_BINARY_DIR}}/{manifest_name}-config.cmake - DESTINATION ${{CMAKE_INSTALL_DIR}} -) -""" - -CMAKE_CONFIG_FILE = """ -@PACKAGE_INIT@ - -include(CMakeFindDependencyMacro) - -set_and_check({upper_name}_CMAKE_DIR "@PACKAGE_CMAKE_INSTALL_DIR@") - -if (NOT TARGET {namespace}::{lib_name}) - include("${{{upper_name}_CMAKE_DIR}}/{manifest_name}-targets.cmake") -endif() - -set({upper_name}_LIBRARIES {namespace}::{lib_name}) - -{find_dependency_lines} - -if (NOT {manifest_name}_FIND_QUIETLY) - message(STATUS "Found {manifest_name}: ${{PACKAGE_PREFIX_DIR}}") -endif() -""" - - -# Note: for now we are manually manipulating the wheel packet contents. -# The wheel format is documented here: -# https://www.python.org/dev/peps/pep-0491/#file-format -# -# We currently aren't particularly smart about correctly handling the full wheel -# functionality, but this is good enough to handle simple pure-python wheels, -# which is the main thing we care about right now. -# -# We could potentially use pip to install the wheel to a temporary location and -# then copy its "installed" files, but this has its own set of complications. -# This would require pip to already be installed and available, and we would -# need to correctly find the right version of pip or pip3 to use. -# If we did ever want to go down that path, we would probably want to use -# something like the following pip3 command: -# pip3 --isolated install --no-cache-dir --no-index --system \ -# --target -class PythonWheelBuilder(BuilderBase): - """This Builder can take Python wheel archives and install them as python libraries - that can be used by add_fb_python_library()/add_fb_python_executable() CMake rules. - """ - - # pyre-fixme[13]: Attribute `dist_info_dir` is never initialized. - dist_info_dir: str - # pyre-fixme[13]: Attribute `template_format_dict` is never initialized. - template_format_dict: Dict[str, str] - - def _build(self, reconfigure: bool) -> None: - # When we are invoked, self.src_dir contains the unpacked wheel contents. - # - # Since a wheel file is just a zip file, the Fetcher code recognizes it as such - # and goes ahead and unpacks it. (We could disable that Fetcher behavior in the - # future if we ever wanted to, say if we wanted to call pip here.) - wheel_name = self._parse_wheel_name() - name_version_prefix = "-".join((wheel_name.distribution, wheel_name.version)) - dist_info_name = name_version_prefix + ".dist-info" - data_dir_name = name_version_prefix + ".data" - self.dist_info_dir = os.path.join(self.src_dir, dist_info_name) - wheel_metadata = self._read_wheel_metadata(wheel_name) - - # Check that we can understand the wheel version. - # We don't really care about wheel_metadata["Root-Is-Purelib"] since - # we are generating our own standalone python archives rather than installing - # into site-packages. - version = wheel_metadata["Wheel-Version"] - if not version.startswith("1."): - raise Exception("unsupported wheel version %s" % (version,)) - - # Add a find_dependency() call for each of our dependencies. - # The dependencies are also listed in the wheel METADATA file, but it is simpler - # to pull this directly from the getdeps manifest. - dep_list = sorted( - self.manifest.get_section_as_dict("dependencies", self.ctx).keys() - ) - find_dependency_lines = ["find_dependency({})".format(dep) for dep in dep_list] - - getdeps_cmake_dir = os.path.join( - os.path.dirname(os.path.dirname(__file__)), "CMake" - ) - self.template_format_dict = { - # Note that CMake files always uses forward slash separators in path names, - # even on Windows. Therefore replace path separators here. - "cmake_dir": _to_cmake_path(getdeps_cmake_dir), - "lib_name": self.manifest.name, - "manifest_name": self.manifest.name, - "namespace": self.manifest.name, - "upper_name": self.manifest.name.upper().replace("-", "_"), - "find_dependency_lines": "\n".join(find_dependency_lines), - } - - # Find sources from the root directory - path_mapping = {} - for entry in os.listdir(self.src_dir): - if entry == data_dir_name: - continue - self._add_sources(path_mapping, os.path.join(self.src_dir, entry), entry) - - # Files under the .data directory also need to be installed in the correct - # locations - if os.path.exists(data_dir_name): - # TODO: process the subdirectories of data_dir_name - # This isn't implemented yet since for now we have only needed dependencies - # on some simple pure Python wheels, so I haven't tested against wheels with - # additional files in the .data directory. - raise Exception( - "handling of the subdirectories inside %s is not implemented yet" - % data_dir_name - ) - - # Emit CMake files - self._write_cmakelists(path_mapping, dep_list) - self._write_cmake_config_template() - - # Run the build - self._run_cmake_build(reconfigure) - - def _run_cmake_build(self, reconfigure: bool) -> None: - cmake_builder = CMakeBuilder( - loader=self.loader, - dep_manifests=self.dep_manifests, - build_opts=self.build_opts, - ctx=self.ctx, - manifest=self.manifest, - # Note that we intentionally supply src_dir=build_dir, - # since we wrote out our generated CMakeLists.txt in the build directory - src_dir=self.build_dir, - build_dir=self.build_dir, - inst_dir=self.inst_dir, - defines={}, - final_install_prefix=None, - ) - cmake_builder.build(reconfigure=reconfigure) - - def _write_cmakelists(self, path_mapping: Dict[str, str], dependencies) -> None: - cmake_path = os.path.join(self.build_dir, "CMakeLists.txt") - with open(cmake_path, "w") as f: - f.write(CMAKE_HEADER.format(**self.template_format_dict)) - for dep in dependencies: - f.write("find_package({0} REQUIRED)\n".format(dep)) - - f.write( - "add_fb_python_library({lib_name}\n".format(**self.template_format_dict) - ) - f.write(' BASE_DIR "%s"\n' % _to_cmake_path(self.src_dir)) - f.write(" SOURCES\n") - for src_path, install_path in path_mapping.items(): - f.write( - ' "%s=%s"\n' - % (_to_cmake_path(src_path), _to_cmake_path(install_path)) - ) - if dependencies: - f.write(" DEPENDS\n") - for dep in dependencies: - f.write(' "{0}::{0}"\n'.format(dep)) - f.write(")\n") - - f.write(CMAKE_FOOTER.format(**self.template_format_dict)) - - def _write_cmake_config_template(self) -> None: - config_path_name = self.manifest.name + "-config.cmake.in" - output_path = os.path.join(self.build_dir, config_path_name) - - with open(output_path, "w") as f: - f.write(CMAKE_CONFIG_FILE.format(**self.template_format_dict)) - - def _add_sources( - self, path_mapping: Dict[str, str], src_path: str, install_path: str - ) -> None: - s = os.lstat(src_path) - if not stat.S_ISDIR(s.st_mode): - path_mapping[src_path] = install_path - return - - for entry in os.listdir(src_path): - self._add_sources( - path_mapping, - os.path.join(src_path, entry), - os.path.join(install_path, entry), - ) - - def _parse_wheel_name(self) -> WheelNameInfo: - # The ArchiveFetcher prepends "manifest_name-", so strip that off first. - wheel_name = os.path.basename(self.src_dir) - prefix = self.manifest.name + "-" - if not wheel_name.startswith(prefix): - raise Exception( - "expected wheel source directory to be of the form %s-NAME.whl" - % (prefix,) - ) - wheel_name = wheel_name[len(prefix) :] - - wheel_name_re = re.compile( - r"(?P[^-]+)" - r"-(?P\d+[^-]*)" - r"(-(?P\d+[^-]*))?" - r"-(?P\w+\d+(\.\w+\d+)*)" - r"-(?P\w+)" - r"-(?P\w+(\.\w+)*)" - r"\.whl" - ) - match = wheel_name_re.match(wheel_name) - if not match: - raise Exception( - "bad python wheel name %s: expected to have the form " - "DISTRIBUTION-VERSION-[-BUILD]-PYTAG-ABI-PLATFORM" - ) - - return WheelNameInfo( - distribution=match.group("distribution"), - version=match.group("version"), - build=match.group("build"), - python=match.group("python"), - abi=match.group("abi"), - platform=match.group("platform"), - ) - - def _read_wheel_metadata(self, wheel_name): - metadata_path = os.path.join(self.dist_info_dir, "WHEEL") - with codecs.open(metadata_path, "r", encoding="utf-8") as f: - return email.message_from_file(f) - - -def _to_cmake_path(path): - # CMake always uses forward slashes to separate paths in CMakeLists.txt files, - # even on Windows. It treats backslashes as character escapes, so using - # backslashes in the path will cause problems. Therefore replace all path - # separators with forward slashes to make sure the paths are correct on Windows. - # e.g. "C:\foo\bar.txt" becomes "C:/foo/bar.txt" - return path.replace(os.path.sep, "/") diff --git a/build/fbcode_builder/getdeps/runcmd.py b/build/fbcode_builder/getdeps/runcmd.py deleted file mode 100644 index 11a13b55f..000000000 --- a/build/fbcode_builder/getdeps/runcmd.py +++ /dev/null @@ -1,170 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-unsafe - -import os -import select -import subprocess -import sys -from shlex import quote as shellquote - -from .envfuncs import Env -from .platform import is_windows - - -class RunCommandError(Exception): - pass - - -def _print_env_diff(env, log_fn) -> None: - current_keys = set(os.environ.keys()) - wanted_env = set(env.keys()) - - unset_keys = current_keys.difference(wanted_env) - for k in sorted(unset_keys): - log_fn("+ unset %s\n" % k) - - added_keys = wanted_env.difference(current_keys) - for k in wanted_env.intersection(current_keys): - if os.environ[k] != env[k]: - added_keys.add(k) - - for k in sorted(added_keys): - if ("PATH" in k) and (os.pathsep in env[k]): - log_fn("+ %s=\\\n" % k) - for elem in env[k].split(os.pathsep): - log_fn("+ %s%s\\\n" % (shellquote(elem), os.pathsep)) - else: - log_fn("+ %s=%s \\\n" % (k, shellquote(env[k]))) - - -def check_cmd(cmd, **kwargs) -> None: - """Run the command and abort on failure""" - rc = run_cmd(cmd, **kwargs) - if rc != 0: - raise RuntimeError(f"Failure exit code {rc} for command {cmd}") - - -def run_cmd(cmd, env=None, cwd=None, allow_fail: bool = False, log_file=None) -> int: - def log_to_stdout(msg): - sys.stdout.buffer.write(msg.encode(errors="surrogateescape")) - - if log_file is not None: - with open(log_file, "a", encoding="utf-8", errors="surrogateescape") as log: - - def log_function(msg): - log.write(msg) - log_to_stdout(msg) - - return _run_cmd( - cmd, env=env, cwd=cwd, allow_fail=allow_fail, log_fn=log_function - ) - else: - return _run_cmd( - cmd, env=env, cwd=cwd, allow_fail=allow_fail, log_fn=log_to_stdout - ) - - -def _run_cmd(cmd, env, cwd, allow_fail, log_fn) -> int: - log_fn("---\n") - try: - cmd_str = " \\\n+ ".join(shellquote(arg) for arg in cmd) - except TypeError: - # eg: one of the elements is None - raise RunCommandError("problem quoting cmd: %r" % cmd) - - if env: - assert isinstance(env, Env) - _print_env_diff(env, log_fn) - - # Convert from our Env type to a regular dict. - # This is needed because python3 looks up b'PATH' and 'PATH' - # and emits an error if both are present. In our Env type - # we'll return the same value for both requests, but we don't - # have duplicate potentially conflicting values which is the - # spirit of the check. - env = dict(env.items()) - - if cwd: - log_fn("+ cd %s && \\\n" % shellquote(cwd)) - # Our long path escape sequence may confuse cmd.exe, so if the cwd - # is short enough, strip that off. - if is_windows() and (len(cwd) < 250) and cwd.startswith("\\\\?\\"): - cwd = cwd[4:] - - log_fn("+ %s\n" % cmd_str) - - isinteractive = os.isatty(sys.stdout.fileno()) - if isinteractive: - stdout = None - sys.stdout.buffer.flush() - else: - stdout = subprocess.PIPE - - try: - p = subprocess.Popen( - cmd, env=env, cwd=cwd, stdout=stdout, stderr=subprocess.STDOUT - ) - except (TypeError, ValueError, OSError) as exc: - log_fn("error running `%s`: %s" % (cmd_str, exc)) - raise RunCommandError( - "%s while running `%s` with env=%r\nos.environ=%r" - % (str(exc), cmd_str, env, os.environ) - ) - - if not isinteractive: - _pipe_output(p, log_fn) - - p.wait() - if p.returncode != 0 and not allow_fail: - raise subprocess.CalledProcessError(p.returncode, cmd) - - return p.returncode - - -if hasattr(select, "poll"): - - def _pipe_output(p, log_fn): - """Read output from p.stdout and call log_fn() with each chunk of data as it - becomes available.""" - # Perform non-blocking reads - import fcntl - - fcntl.fcntl(p.stdout.fileno(), fcntl.F_SETFL, os.O_NONBLOCK) - poll = select.poll() - poll.register(p.stdout.fileno(), select.POLLIN) - - buffer_size = 4096 - while True: - poll.poll() - data = p.stdout.read(buffer_size) - if not data: - break - # log_fn() accepts arguments as str (binary in Python 2, unicode in - # Python 3). In Python 3 the subprocess output will be plain bytes, - # and need to be decoded. - if not isinstance(data, str): - data = data.decode("utf-8", errors="surrogateescape") - log_fn(data) - -else: - - def _pipe_output(p, log_fn): - """Read output from p.stdout and call log_fn() with each chunk of data as it - becomes available.""" - # Perform blocking reads. Use a smaller buffer size to avoid blocking - # for very long when data is available. - buffer_size = 64 - while True: - data = p.stdout.read(buffer_size) - if not data: - break - # log_fn() accepts arguments as str (binary in Python 2, unicode in - # Python 3). In Python 3 the subprocess output will be plain bytes, - # and need to be decoded. - if not isinstance(data, str): - data = data.decode("utf-8", errors="surrogateescape") - log_fn(data) diff --git a/build/fbcode_builder/getdeps/subcmd.py b/build/fbcode_builder/getdeps/subcmd.py deleted file mode 100644 index acbeb93f1..000000000 --- a/build/fbcode_builder/getdeps/subcmd.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-unsafe - - -class SubCmd(object): - NAME = None - HELP = None - - def run(self, args) -> int: - """perform the command""" - return 0 - - def setup_parser(self, parser) -> None: - # Subclasses should override setup_parser() if they have any - # command line options or arguments. - pass - - -CmdTable = [] - - -def add_subcommands(parser, common_args, cmd_table=CmdTable) -> None: - """Register parsers for the defined commands with the provided parser""" - for cls in cmd_table: - command = cls() - command_parser = parser.add_parser( - command.NAME, help=command.HELP, parents=[common_args] - ) - command.setup_parser(command_parser) - command_parser.set_defaults(func=command.run) - - -def cmd(name, help=None, cmd_table=CmdTable): - """ - @cmd() is a decorator that can be used to help define Subcmd instances - - Example usage: - - @subcmd('list', 'Show the result list') - class ListCmd(Subcmd): - def run(self, args): - # Perform the command actions here... - pass - """ - - def wrapper(cls): - class SubclassedCmd(cls): - NAME = name - HELP = help - - cmd_table.append(SubclassedCmd) - return SubclassedCmd - - return wrapper diff --git a/build/fbcode_builder/getdeps/test/expr_test.py b/build/fbcode_builder/getdeps/test/expr_test.py deleted file mode 100644 index 4f4b957ce..000000000 --- a/build/fbcode_builder/getdeps/test/expr_test.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-unsafe - - -import unittest - -from ..expr import parse_expr - - -class ExprTest(unittest.TestCase): - def test_equal(self) -> None: - valid_variables = {"foo", "some_var", "another_var"} - e = parse_expr("foo=bar", valid_variables) - self.assertTrue(e.eval({"foo": "bar"})) - self.assertFalse(e.eval({"foo": "not-bar"})) - self.assertFalse(e.eval({"not-foo": "bar"})) - - def test_not_equal(self) -> None: - valid_variables = {"foo"} - e = parse_expr("not(foo=bar)", valid_variables) - self.assertFalse(e.eval({"foo": "bar"})) - self.assertTrue(e.eval({"foo": "not-bar"})) - - def test_bad_not(self) -> None: - valid_variables = {"foo"} - with self.assertRaises(Exception): - parse_expr("foo=not(bar)", valid_variables) - - def test_bad_variable(self) -> None: - valid_variables = {"bar"} - with self.assertRaises(Exception): - parse_expr("foo=bar", valid_variables) - - def test_all(self) -> None: - valid_variables = {"foo", "baz"} - e = parse_expr("all(foo = bar, baz = qux)", valid_variables) - self.assertTrue(e.eval({"foo": "bar", "baz": "qux"})) - self.assertFalse(e.eval({"foo": "bar", "baz": "nope"})) - self.assertFalse(e.eval({"foo": "nope", "baz": "nope"})) - - def test_any(self) -> None: - valid_variables = {"foo", "baz"} - e = parse_expr("any(foo = bar, baz = qux)", valid_variables) - self.assertTrue(e.eval({"foo": "bar", "baz": "qux"})) - self.assertTrue(e.eval({"foo": "bar", "baz": "nope"})) - self.assertFalse(e.eval({"foo": "nope", "baz": "nope"})) diff --git a/build/fbcode_builder/getdeps/test/fixtures/duplicate/foo b/build/fbcode_builder/getdeps/test/fixtures/duplicate/foo deleted file mode 100644 index a0384ee3b..000000000 --- a/build/fbcode_builder/getdeps/test/fixtures/duplicate/foo +++ /dev/null @@ -1,2 +0,0 @@ -[manifest] -name = foo diff --git a/build/fbcode_builder/getdeps/test/fixtures/duplicate/subdir/foo b/build/fbcode_builder/getdeps/test/fixtures/duplicate/subdir/foo deleted file mode 100644 index a0384ee3b..000000000 --- a/build/fbcode_builder/getdeps/test/fixtures/duplicate/subdir/foo +++ /dev/null @@ -1,2 +0,0 @@ -[manifest] -name = foo diff --git a/build/fbcode_builder/getdeps/test/manifest_test.py b/build/fbcode_builder/getdeps/test/manifest_test.py deleted file mode 100644 index 2bb133196..000000000 --- a/build/fbcode_builder/getdeps/test/manifest_test.py +++ /dev/null @@ -1,234 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-unsafe - - -import sys -import unittest - -from ..load import load_all_manifests, patch_loader -from ..manifest import ManifestParser - - -class ManifestTest(unittest.TestCase): - def test_missing_section(self) -> None: - with self.assertRaisesRegex( - Exception, "manifest file test is missing required section manifest" - ): - ManifestParser("test", "") - - def test_missing_name(self) -> None: - with self.assertRaisesRegex( - Exception, - "manifest file test section 'manifest' is missing required field 'name'", - ): - ManifestParser( - "test", - """ -[manifest] -""", - ) - - def test_minimal(self) -> None: - p = ManifestParser( - "test", - """ -[manifest] -name = test -""", - ) - self.assertEqual(p.name, "test") - self.assertEqual(p.fbsource_path, None) - - def test_minimal_with_fbsource_path(self) -> None: - p = ManifestParser( - "test", - """ -[manifest] -name = test -fbsource_path = fbcode/wat -""", - ) - self.assertEqual(p.name, "test") - self.assertEqual(p.fbsource_path, "fbcode/wat") - - def test_unknown_field(self) -> None: - with self.assertRaisesRegex( - Exception, - ( - "manifest file test section 'manifest' contains " - "unknown field 'invalid.field'" - ), - ): - ManifestParser( - "test", - """ -[manifest] -name = test -invalid.field = woot -""", - ) - - def test_invalid_section_name(self) -> None: - with self.assertRaisesRegex( - Exception, "manifest file test contains unknown section 'invalid.section'" - ): - ManifestParser( - "test", - """ -[manifest] -name = test - -[invalid.section] -foo = bar -""", - ) - - def test_value_in_dependencies_section(self) -> None: - with self.assertRaisesRegex( - Exception, - ( - "manifest file test section 'dependencies' has " - "'foo = bar' but this section doesn't allow " - "specifying values for its entries" - ), - ): - ManifestParser( - "test", - """ -[manifest] -name = test - -[dependencies] -foo = bar -""", - ) - - def test_invalid_conditional_section_name(self) -> None: - with self.assertRaisesRegex( - Exception, - ( - "manifest file test section 'dependencies.=' " - "has invalid conditional: expected " - "identifier found =" - ), - ): - ManifestParser( - "test", - """ -[manifest] -name = test - -[dependencies.=] -""", - ) - - def test_section_as_args(self) -> None: - p = ManifestParser( - "test", - """ -[manifest] -name = test - -[dependencies] -a -b -c - -[dependencies.test=on] -foo -""", - ) - self.assertEqual(p.get_section_as_args("dependencies"), ["a", "b", "c"]) - self.assertEqual( - p.get_section_as_args("dependencies", {"test": "off"}), ["a", "b", "c"] - ) - self.assertEqual( - p.get_section_as_args("dependencies", {"test": "on"}), - ["a", "b", "c", "foo"], - ) - - p2 = ManifestParser( - "test", - """ -[manifest] -name = test - -[autoconf.args] ---prefix=/foo ---with-woot -""", - ) - self.assertEqual( - p2.get_section_as_args("autoconf.args"), ["--prefix=/foo", "--with-woot"] - ) - - def test_section_as_dict(self) -> None: - p = ManifestParser( - "test", - """ -[manifest] -name = test - -[cmake.defines] -foo = bar - -[cmake.defines.test=on] -foo = baz -""", - ) - self.assertEqual(p.get_section_as_dict("cmake.defines", {}), {"foo": "bar"}) - self.assertEqual( - p.get_section_as_dict("cmake.defines", {"test": "on"}), {"foo": "baz"} - ) - - p2 = ManifestParser( - "test", - """ -[manifest] -name = test - -[cmake.defines.test=on] -foo = baz - -[cmake.defines] -foo = bar -""", - ) - self.assertEqual( - p2.get_section_as_dict("cmake.defines", {"test": "on"}), - {"foo": "bar"}, - msg="sections cascade in the order they appear in the manifest", - ) - - def test_parse_common_manifests(self) -> None: - patch_loader(__name__) - manifests = load_all_manifests(None) - self.assertNotEqual(0, len(manifests), msg="parsed some number of manifests") - - def test_mismatch_name(self) -> None: - with self.assertRaisesRegex( - Exception, - "filename of the manifest 'foo' does not match the manifest name 'bar'", - ): - ManifestParser( - "foo", - """ -[manifest] -name = bar -""", - ) - - def test_duplicate_manifest(self) -> None: - patch_loader(__name__, "fixtures/duplicate") - - with self.assertRaisesRegex(Exception, "found duplicate manifest 'foo'"): - load_all_manifests(None) - - if sys.version_info < (3, 2): - - def assertRaisesRegex(self, *args, **kwargs): - return self.assertRaisesRegex(*args, **kwargs) diff --git a/build/fbcode_builder/getdeps/test/platform_test.py b/build/fbcode_builder/getdeps/test/platform_test.py deleted file mode 100644 index 1fcab7a58..000000000 --- a/build/fbcode_builder/getdeps/test/platform_test.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-unsafe - - -import unittest - -from ..platform import HostType - - -class PlatformTest(unittest.TestCase): - def test_create(self) -> None: - p = HostType() - self.assertNotEqual(p.ostype, None, msg="probed and returned something") - - tuple_string = p.as_tuple_string() - round_trip = HostType.from_tuple_string(tuple_string) - self.assertEqual(round_trip, p) - - def test_rendering_of_none(self) -> None: - p = HostType(ostype="foo") - self.assertEqual(p.as_tuple_string(), "foo-none-none") - - def test_is_methods(self) -> None: - p = HostType(ostype="windows") - self.assertTrue(p.is_windows()) - self.assertFalse(p.is_darwin()) - self.assertFalse(p.is_linux()) - - p = HostType(ostype="darwin") - self.assertFalse(p.is_windows()) - self.assertTrue(p.is_darwin()) - self.assertFalse(p.is_linux()) - - p = HostType(ostype="linux") - self.assertFalse(p.is_windows()) - self.assertFalse(p.is_darwin()) - self.assertTrue(p.is_linux()) diff --git a/build/fbcode_builder/getdeps/test/retry_test.py b/build/fbcode_builder/getdeps/test/retry_test.py deleted file mode 100644 index f1d159832..000000000 --- a/build/fbcode_builder/getdeps/test/retry_test.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-unsafe - - -import unittest -from unittest.mock import call, MagicMock, patch - -from ..buildopts import BuildOptions -from ..errors import TransientFailure -from ..fetcher import ArchiveFetcher -from ..manifest import ManifestParser - - -class RetryTest(unittest.TestCase): - def _get_build_opts(self) -> BuildOptions: - mock_build_opts = MagicMock(spec=BuildOptions) - mock_build_opts.scratch_dir = "/path/to/scratch_dir" - return mock_build_opts - - def _get_manifest(self) -> ManifestParser: - mock_manifest_parser = MagicMock(spec=ManifestParser) - mock_manifest_parser.name = "mock_manifest_parser" - return mock_manifest_parser - - def _get_archive_fetcher(self) -> ArchiveFetcher: - return ArchiveFetcher( - build_options=self._get_build_opts(), - manifest=self._get_manifest(), - url="https://github.com/systemd/systemd/archive/refs/tags/v256.7.tar.gz", - sha256="896d76ff65c88f5fd9e42f90d152b0579049158a163431dd77cdc57748b1d7b0", - ) - - @patch("os.makedirs") - @patch("os.environ.get") - @patch("time.sleep") - @patch("subprocess.run") - def test_no_retries( - self, mock_run, mock_sleep, mock_os_environ_get, mock_makedirs - ) -> None: - def custom_makedirs(path, exist_ok=False): - return None - - def custom_get(key, default=None): - if key == "GETDEPS_USE_WGET": - return "1" - elif key == "GETDEPS_WGET_ARGS": - return "" - else: - return None - - mock_makedirs.side_effect = custom_makedirs - mock_os_environ_get.side_effect = custom_get - mock_sleep.side_effect = None - fetcher = self._get_archive_fetcher() - fetcher._verify_hash = MagicMock(return_value=None) - fetcher._download() - mock_sleep.assert_has_calls([], any_order=False) - mock_run.assert_called_once_with( - [ - "wget", - "-O", - "/path/to/scratch_dir/downloads/mock_manifest_parser-v256.7.tar.gz", - "https://github.com/systemd/systemd/archive/refs/tags/v256.7.tar.gz", - ], - capture_output=True, - ) - - @patch("random.random") - @patch("os.makedirs") - @patch("os.environ.get") - @patch("time.sleep") - @patch("subprocess.run") - def test_retries( - self, mock_run, mock_sleep, mock_os_environ_get, mock_makedirs, mock_random - ) -> None: - def custom_makedirs(path, exist_ok=False): - return None - - def custom_get(key, default=None): - if key == "GETDEPS_USE_WGET": - return "1" - elif key == "GETDEPS_WGET_ARGS": - return "" - else: - return None - - mock_random.return_value = 0 - - mock_run.side_effect = [ - IOError(""), - IOError(""), - None, - ] - mock_makedirs.side_effect = custom_makedirs - mock_os_environ_get.side_effect = custom_get - mock_sleep.side_effect = None - fetcher = self._get_archive_fetcher() - fetcher._verify_hash = MagicMock(return_value=None) - fetcher._download() - mock_sleep.assert_has_calls([call(2), call(4)], any_order=False) - calls = [ - call( - [ - "wget", - "-O", - "/path/to/scratch_dir/downloads/mock_manifest_parser-v256.7.tar.gz", - "https://github.com/systemd/systemd/archive/refs/tags/v256.7.tar.gz", - ], - capture_output=True, - ), - ] * 3 - - mock_run.assert_has_calls(calls, any_order=False) - - @patch("random.random") - @patch("os.makedirs") - @patch("os.environ.get") - @patch("time.sleep") - @patch("subprocess.run") - def test_all_retries( - self, mock_run, mock_sleep, mock_os_environ_get, mock_makedirs, mock_random - ) -> None: - def custom_makedirs(path, exist_ok=False): - return None - - def custom_get(key, default=None): - if key == "GETDEPS_USE_WGET": - return "1" - elif key == "GETDEPS_WGET_ARGS": - return "" - else: - return None - - mock_random.return_value = 0 - - mock_run.side_effect = IOError( - "" - ) - mock_makedirs.side_effect = custom_makedirs - mock_os_environ_get.side_effect = custom_get - mock_sleep.side_effect = None - fetcher = self._get_archive_fetcher() - fetcher._verify_hash = MagicMock(return_value=None) - with self.assertRaises(TransientFailure): - fetcher._download() - mock_sleep.assert_has_calls( - [call(2), call(4), call(8), call(10)], any_order=False - ) - calls = [ - call( - [ - "wget", - "-O", - "/path/to/scratch_dir/downloads/mock_manifest_parser-v256.7.tar.gz", - "https://github.com/systemd/systemd/archive/refs/tags/v256.7.tar.gz", - ], - capture_output=True, - ), - ] * 5 - - mock_run.assert_has_calls(calls, any_order=False) diff --git a/build/fbcode_builder/getdeps/test/scratch_test.py b/build/fbcode_builder/getdeps/test/scratch_test.py deleted file mode 100644 index 4075e0a3d..000000000 --- a/build/fbcode_builder/getdeps/test/scratch_test.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -# pyre-unsafe - - -import unittest - -from ..buildopts import find_existing_win32_subst_for_path - - -class Win32SubstTest(unittest.TestCase): - def test_no_existing_subst(self) -> None: - self.assertIsNone( - find_existing_win32_subst_for_path( - r"C:\users\alice\appdata\local\temp\fbcode_builder_getdeps", - subst_mapping={}, - ) - ) - self.assertIsNone( - find_existing_win32_subst_for_path( - r"C:\users\alice\appdata\local\temp\fbcode_builder_getdeps", - subst_mapping={"X:\\": r"C:\users\alice\appdata\local\temp\other"}, - ) - ) - - def test_exact_match_returns_drive_path(self) -> None: - self.assertEqual( - find_existing_win32_subst_for_path( - r"C:\temp\fbcode_builder_getdeps", - subst_mapping={"X:\\": r"C:\temp\fbcode_builder_getdeps"}, - ), - "X:\\", - ) - self.assertEqual( - find_existing_win32_subst_for_path( - r"C:/temp/fbcode_builder_getdeps", - subst_mapping={"X:\\": r"C:/temp/fbcode_builder_getdeps"}, - ), - "X:\\", - ) - - def test_multiple_exact_matches_returns_arbitrary_drive_path(self) -> None: - self.assertIn( - find_existing_win32_subst_for_path( - r"C:\temp\fbcode_builder_getdeps", - subst_mapping={ - "X:\\": r"C:\temp\fbcode_builder_getdeps", - "Y:\\": r"C:\temp\fbcode_builder_getdeps", - "Z:\\": r"C:\temp\fbcode_builder_getdeps", - }, - ), - ("X:\\", "Y:\\", "Z:\\"), - ) - - def test_drive_letter_is_case_insensitive(self) -> None: - self.assertEqual( - find_existing_win32_subst_for_path( - r"C:\temp\fbcode_builder_getdeps", - subst_mapping={"X:\\": r"c:\temp\fbcode_builder_getdeps"}, - ), - "X:\\", - ) - - def test_path_components_are_case_insensitive(self) -> None: - self.assertEqual( - find_existing_win32_subst_for_path( - r"C:\TEMP\FBCODE_builder_getdeps", - subst_mapping={"X:\\": r"C:\temp\fbcode_builder_getdeps"}, - ), - "X:\\", - ) - self.assertEqual( - find_existing_win32_subst_for_path( - r"C:\temp\fbcode_builder_getdeps", - subst_mapping={"X:\\": r"C:\TEMP\FBCODE_builder_getdeps"}, - ), - "X:\\", - ) diff --git a/build/fbcode_builder/manifests/CLI11 b/build/fbcode_builder/manifests/CLI11 deleted file mode 100644 index 14cb2332a..000000000 --- a/build/fbcode_builder/manifests/CLI11 +++ /dev/null @@ -1,14 +0,0 @@ -[manifest] -name = CLI11 - -[download] -url = https://github.com/CLIUtils/CLI11/archive/v2.0.0.tar.gz -sha256 = 2c672f17bf56e8e6223a3bfb74055a946fa7b1ff376510371902adb9cb0ab6a3 - -[build] -builder = cmake -subdir = CLI11-2.0.0 - -[cmake.defines] -CLI11_BUILD_TESTS = OFF -CLI11_BUILD_EXAMPLES = OFF diff --git a/build/fbcode_builder/manifests/autoconf b/build/fbcode_builder/manifests/autoconf deleted file mode 100644 index 444daa444..000000000 --- a/build/fbcode_builder/manifests/autoconf +++ /dev/null @@ -1,22 +0,0 @@ -[manifest] -name = autoconf - -[debs] -autoconf - -[homebrew] -autoconf - -[rpms] -autoconf - -[pps] -autoconf - -[download] -url = https://ftpmirror.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz -sha256 = 954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969 - -[build] -builder = autoconf -subdir = autoconf-2.69 diff --git a/build/fbcode_builder/manifests/automake b/build/fbcode_builder/manifests/automake deleted file mode 100644 index 857a9ca61..000000000 --- a/build/fbcode_builder/manifests/automake +++ /dev/null @@ -1,25 +0,0 @@ -[manifest] -name = automake - -[homebrew] -automake - -[debs] -automake - -[rpms] -automake - -[pps] -automake - -[download] -url = https://ftpmirror.gnu.org/gnu/automake/automake-1.16.1.tar.gz -sha256 = 608a97523f97db32f1f5d5615c98ca69326ced2054c9f82e65bade7fc4c9dea8 - -[build] -builder = autoconf -subdir = automake-1.16.1 - -[dependencies] -autoconf diff --git a/build/fbcode_builder/manifests/benchmark b/build/fbcode_builder/manifests/benchmark deleted file mode 100644 index 25d621184..000000000 --- a/build/fbcode_builder/manifests/benchmark +++ /dev/null @@ -1,13 +0,0 @@ -[manifest] -name = benchmark - -[download] -url = https://github.com/google/benchmark/archive/refs/tags/v1.8.0.tar.gz -sha256 = ea2e94c24ddf6594d15c711c06ccd4486434d9cf3eca954e2af8a20c88f9f172 - -[build] -builder = cmake -subdir = benchmark-1.8.0/ - -[cmake.defines] -BENCHMARK_ENABLE_TESTING=OFF diff --git a/build/fbcode_builder/manifests/blake3 b/build/fbcode_builder/manifests/blake3 deleted file mode 100644 index 12ee6518f..000000000 --- a/build/fbcode_builder/manifests/blake3 +++ /dev/null @@ -1,10 +0,0 @@ -[manifest] -name = blake3 - -[download] -url = https://github.com/BLAKE3-team/BLAKE3/archive/refs/tags/1.5.1.tar.gz -sha256 = 822cd37f70152e5985433d2c50c8f6b2ec83aaf11aa31be9fe71486a91744f37 - -[build] -builder = cmake -subdir = BLAKE3-1.5.1/c diff --git a/build/fbcode_builder/manifests/boost b/build/fbcode_builder/manifests/boost deleted file mode 100644 index 97323093a..000000000 --- a/build/fbcode_builder/manifests/boost +++ /dev/null @@ -1,116 +0,0 @@ -[manifest] -name = boost - -[download.not(os=windows)] -url = https://archives.boost.io/release/1.83.0/source/boost_1_83_0.tar.gz -sha256 = c0685b68dd44cc46574cce86c4e17c0f611b15e195be9848dfd0769a0a207628 - -[download.os=windows] -url = https://archives.boost.io/release/1.83.0/source/boost_1_83_0.zip -sha256 = c86bd9d9eef795b4b0d3802279419fde5221922805b073b9bd822edecb1ca28e - -[preinstalled.env] -# Here we list the acceptable versions that cmake needs a hint to find -BOOST_ROOT_1_69_0 -BOOST_ROOT_1_83_0 - -[debs] -libboost-all-dev - -[homebrew] -boost -# Boost cmake detection on homebrew adds this as requirement: https://github.com/Homebrew/homebrew-core/issues/67427#issuecomment-754187345 -icu4c - -[pps] -boost - -[rpms.all(distro=centos_stream,distro_vers=8)] -boost169 -boost169-math -boost169-test -boost169-fiber -boost169-graph -boost169-log -boost169-openmpi -boost169-timer -boost169-chrono -boost169-locale -boost169-thread -boost169-atomic -boost169-random -boost169-static -boost169-contract -boost169-date-time -boost169-iostreams -boost169-container -boost169-coroutine -boost169-filesystem -boost169-system -boost169-stacktrace -boost169-regex -boost169-devel -boost169-context -boost169-python3-devel -boost169-type_erasure -boost169-wave -boost169-python3 -boost169-serialization -boost169-program-options - -[rpms.distro=fedora] -boost-devel -boost-static - -[build] -builder = boost -job_weight_mib = 512 -patchfile = boost_1_83_0.patch - -[b2.args] ---with-atomic ---with-chrono ---with-container ---with-context ---with-contract ---with-coroutine ---with-date_time ---with-exception ---with-fiber ---with-filesystem ---with-graph ---with-graph_parallel ---with-iostreams ---with-locale ---with-log ---with-math ---with-mpi ---with-program_options ---with-python ---with-random ---with-regex ---with-serialization ---with-stacktrace ---with-system ---with-test ---with-thread ---with-timer ---with-type_erasure - -[bootstrap.args.os=darwin] -# Not really gcc, but CI puts a broken clang in the PATH, and saying gcc -# here selects the correct one from Xcode. ---with-toolset=gcc - -[b2.args.os=linux] -# RHEL hardened gcc is not compatible with PCH -# https://bugzilla.redhat.com/show_bug.cgi?id=1806545 -pch=off - -[b2.args.os=darwin] -toolset=clang -# Since Xcode 15.3 std::piecewise_construct is only visible in C++17 and later modes -cxxflags="-DBOOST_UNORDERED_HAVE_PIECEWISE_CONSTRUCT=0" - -[b2.args.all(os=windows,fb=on)] -toolset=msvc-14.3 diff --git a/build/fbcode_builder/manifests/boost-python b/build/fbcode_builder/manifests/boost-python deleted file mode 100644 index 8e1c6c5d4..000000000 --- a/build/fbcode_builder/manifests/boost-python +++ /dev/null @@ -1,118 +0,0 @@ -[manifest] -name = boost-python - -[download.not(os=windows)] -url = https://archives.boost.io/release/1.83.0/source/boost_1_83_0.tar.gz -sha256 = c0685b68dd44cc46574cce86c4e17c0f611b15e195be9848dfd0769a0a207628 - -[download.os=windows] -url = https://archives.boost.io/release/1.83.0/source/boost_1_83_0.zip -sha256 = c86bd9d9eef795b4b0d3802279419fde5221922805b073b9bd822edecb1ca28e - -[preinstalled.env] -# Here we list the acceptable versions that cmake needs a hint to find -BOOST_ROOT_1_69_0 -BOOST_ROOT_1_83_0 - -[homebrew] -boost -# Boost cmake detection on homebrew adds this as requirement: https://github.com/Homebrew/homebrew-core/issues/67427#issuecomment-754187345 -icu4c - -[pps] -boost - -[rpms.all(distro=centos_stream,distro_vers=8)] -boost169 -boost169-math -boost169-test -boost169-fiber -boost169-graph -boost169-log -boost169-openmpi -boost169-timer -boost169-chrono -boost169-locale -boost169-thread -boost169-atomic -boost169-random -boost169-static -boost169-contract -boost169-date-time -boost169-iostreams -boost169-container -boost169-coroutine -boost169-filesystem -boost169-system -boost169-stacktrace -boost169-regex -boost169-devel -boost169-context -boost169-python3-devel -boost169-type_erasure -boost169-wave -boost169-python3 -boost169-serialization -boost169-program-options - -[rpms.distro=fedora] -boost-devel -boost-static - -[build] -builder = boost -job_weight_mib = 512 -patchfile = boost_1_83_0.patch - -[build.not(os=linux)] -builder = nop - -[b2.args] ---with-atomic ---with-chrono ---with-container ---with-context ---with-contract ---with-coroutine ---with-date_time ---with-exception ---with-fiber ---with-filesystem ---with-graph ---with-graph_parallel ---with-iostreams ---with-locale ---with-log ---with-math ---with-mpi ---with-program_options ---with-python ---with-random ---with-regex ---with-serialization ---with-stacktrace ---with-system ---with-test ---with-thread ---with-timer ---with-type_erasure - -[bootstrap.args.os=darwin] -# Not really gcc, but CI puts a broken clang in the PATH, and saying gcc -# here selects the correct one from Xcode. ---with-toolset=gcc - -[b2.args.os=linux] -# RHEL hardened gcc is not compatible with PCH -# https://bugzilla.redhat.com/show_bug.cgi?id=1806545 -pch=off -# Python extensions need -fPIC for static library linking into shared objects -cxxflags="-fPIC" - -[b2.args.os=darwin] -toolset=clang -# Since Xcode 15.3 std::piecewise_construct is only visible in C++17 and later modes -cxxflags="-DBOOST_UNORDERED_HAVE_PIECEWISE_CONSTRUCT=0" - -[b2.args.all(os=windows,fb=on)] -toolset=msvc-14.3 diff --git a/build/fbcode_builder/manifests/bz2 b/build/fbcode_builder/manifests/bz2 deleted file mode 100644 index cfbea9c8f..000000000 --- a/build/fbcode_builder/manifests/bz2 +++ /dev/null @@ -1,32 +0,0 @@ -[manifest] -name = bz2 - -[debs] -libbz2-dev -bzip2 - -[homebrew] -bzip2 - -[rpms] -bzip2-devel -bzip2 - -[download] -url = https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz -sha256 = ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269 - -[build.not(os=windows)] -builder = make -subdir = bzip2-1.0.8 - -[make.build_args.os=linux] -# python bz2 support on linux needs dynamic library --f -Makefile-libbz2_so - -[make.install_args] -install - -[build.os=windows] -builder = nop diff --git a/build/fbcode_builder/manifests/c-ares b/build/fbcode_builder/manifests/c-ares deleted file mode 100644 index 7c92bbc0a..000000000 --- a/build/fbcode_builder/manifests/c-ares +++ /dev/null @@ -1,13 +0,0 @@ -[manifest] -name = c-ares - -[download] -url = https://github.com/c-ares/c-ares/releases/download/v1.34.5/c-ares-1.34.5.tar.gz -sha256 = 7d935790e9af081c25c495fd13c2cfcda4792983418e96358ef6e7320ee06346 - -[build] -builder = cmake -subdir = c-ares-1.34.5 - -[cmake.defines] -CARES_STATIC = ON diff --git a/build/fbcode_builder/manifests/cabal b/build/fbcode_builder/manifests/cabal deleted file mode 100644 index 1405b8bc8..000000000 --- a/build/fbcode_builder/manifests/cabal +++ /dev/null @@ -1,12 +0,0 @@ -[manifest] -name = cabal - -[download.os=linux] -url = https://downloads.haskell.org/~cabal/cabal-install-3.6.2.0/cabal-install-3.6.2.0-x86_64-linux-deb10.tar.xz -sha256 = 4759b56e9257e02f29fa374a6b25d6cb2f9d80c7e3a55d4f678a8e570925641c - -[build] -builder = nop - -[install.files] -cabal = bin/cabal diff --git a/build/fbcode_builder/manifests/cachelib b/build/fbcode_builder/manifests/cachelib deleted file mode 100644 index fc6d038b2..000000000 --- a/build/fbcode_builder/manifests/cachelib +++ /dev/null @@ -1,40 +0,0 @@ -[manifest] -name = cachelib -fbsource_path = fbcode/cachelib -shipit_project = cachelib -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebook/cachelib.git - -[build] -builder = cmake -subdir = cachelib -job_weight_mib = 2048 - -[dependencies] -zlib -fizz -fmt -folly -fbthrift -googletest -sparsemap -wangle -zstd -mvfst -numa -libaio -magic_enum -# cachelib also depends on openssl but since the latter requires a platform- -# specific configuration we rely on the folly manifest to provide this -# dependency to avoid duplication. - -[shipit.pathmap] -fbcode/cachelib = cachelib -fbcode/cachelib/public_tld = . - -[shipit.strip] -^fbcode/cachelib/examples(/|$) -^fbcode/cachelib/facebook(/|$) -^fbcode/cachelib/public_tld/website/docs/facebook(/|$) diff --git a/build/fbcode_builder/manifests/cinderx-3_14 b/build/fbcode_builder/manifests/cinderx-3_14 deleted file mode 100644 index c25f8b94d..000000000 --- a/build/fbcode_builder/manifests/cinderx-3_14 +++ /dev/null @@ -1,28 +0,0 @@ -[manifest] -name = cinderx-3_14 -fbsource_path = fbcode/cinderx -shipit_project = facebookincubator/cinderx - -[git] -repo_url = https://github.com/facebookincubator/cinderx.git - -[build.os=linux] -builder = setup-py - -[build.not(os=linux)] -builder = nop - -[dependencies] -python-setuptools -python-3_14 - -[shipit.pathmap] -fbcode/cinderx = cinderx -fbcode/cinderx/oss_toplevel = . - -[setup-py.test] -python_script = cinderx/PythonLib/test_cinderx/test_oss_quick.py - -[setup-py.env] -CINDERX_ENABLE_PGO=1 -CINDERX_ENABLE_LTO=1 diff --git a/build/fbcode_builder/manifests/cinderx-main b/build/fbcode_builder/manifests/cinderx-main deleted file mode 100644 index 053f8b59a..000000000 --- a/build/fbcode_builder/manifests/cinderx-main +++ /dev/null @@ -1,34 +0,0 @@ -# For building CinderX against CPython main. -# Note that externally this can be broken because in that environment we will -# be checking out the head of the CPython repo. However CinderX is only built -# and tested against our internal copy of CPython which updates ~daily, and so -# may be behind CPython head. - -[manifest] -name = cinderx-main -fbsource_path = fbcode/cinderx -shipit_project = facebookincubator/cinderx - -[git] -repo_url = https://github.com/facebookincubator/cinderx.git - -[build.os=linux] -builder = setup-py - -[build.not(os=linux)] -builder = nop - -[dependencies] -python-setuptools -python-main - -[shipit.pathmap] -fbcode/cinderx = cinderx -fbcode/cinderx/oss_toplevel = . - -[setup-py.test] -python_script = cinderx/PythonLib/test_cinderx/test_oss_quick.py - -[setup-py.env] -CINDERX_ENABLE_PGO=1 -CINDERX_ENABLE_LTO=1 diff --git a/build/fbcode_builder/manifests/clang b/build/fbcode_builder/manifests/clang deleted file mode 100644 index a2133e018..000000000 --- a/build/fbcode_builder/manifests/clang +++ /dev/null @@ -1,5 +0,0 @@ -[manifest] -name = clang - -[rpms] -clang15-devel diff --git a/build/fbcode_builder/manifests/cmake b/build/fbcode_builder/manifests/cmake deleted file mode 100644 index 70cffaeca..000000000 --- a/build/fbcode_builder/manifests/cmake +++ /dev/null @@ -1,49 +0,0 @@ -[manifest] -name = cmake - -[homebrew] -cmake - -# 18.04 cmake is too old -[debs.not(all(distro=ubuntu,distro_vers="18.04"))] -cmake - -[rpms] -cmake - -[pps] -cmake - -[dependencies] -ninja - -[download.os=windows] -url = https://github.com/Kitware/CMake/releases/download/v3.20.4/cmake-3.20.4-windows-x86_64.zip -sha256 = 965d2f001c3ca807d288f2b6b15c42b25579a0e73ef12c2a72c95f4c69123638 - -[download.os=darwin] -url = https://github.com/Kitware/CMake/releases/download/v3.20.4/cmake-3.20.4-macos-universal.tar.gz -sha256 = df90016635e3183834143c6d94607f0804fe9762f7cc6032f6a4afd7c19cd43b - -[download.any(os=linux,os=freebsd)] -url = https://github.com/Kitware/CMake/releases/download/v3.20.4/cmake-3.20.4.tar.gz -sha256 = 87a4060298f2c6bb09d479de1400bc78195a5b55a65622a7dceeb3d1090a1b16 - -[build.os=windows] -builder = nop -subdir = cmake-3.20.4-windows-x86_64 - -[build.os=darwin] -builder = nop -subdir = cmake-3.20.4-macos-universal - -[install.files.os=darwin] -CMake.app/Contents/bin = bin -CMake.app/Contents/share = share - -[build.any(os=linux,os=freebsd)] -builder = cmakebootstrap -subdir = cmake-3.20.4 - -[make.install_args.any(os=linux,os=freebsd)] -install diff --git a/build/fbcode_builder/manifests/cpptoml b/build/fbcode_builder/manifests/cpptoml deleted file mode 100644 index c4d6d8d9c..000000000 --- a/build/fbcode_builder/manifests/cpptoml +++ /dev/null @@ -1,16 +0,0 @@ -[manifest] -name = cpptoml - -[homebrew] -cpptoml - -[download] -url = https://github.com/chadaustin/cpptoml/archive/refs/tags/v0.1.2.tar.gz -sha256 = beda37e94f9746874436c8090c045fd80ae6f8a51f7c668c932a2b110a4fc277 - -[build] -builder = cmake -subdir = cpptoml-0.1.2 - -[cmake.defines.os=freebsd] -ENABLE_LIBCXX=NO diff --git a/build/fbcode_builder/manifests/double-conversion b/build/fbcode_builder/manifests/double-conversion deleted file mode 100644 index 720d9a2ec..000000000 --- a/build/fbcode_builder/manifests/double-conversion +++ /dev/null @@ -1,23 +0,0 @@ -[manifest] -name = double-conversion - -[download] -url = https://github.com/google/double-conversion/archive/v3.1.4.tar.gz -sha256 = 95004b65e43fefc6100f337a25da27bb99b9ef8d4071a36a33b5e83eb1f82021 - -[homebrew] -double-conversion - -[debs] -libdouble-conversion-dev - -[rpms] -double-conversion -double-conversion-devel - -[pps] -double-conversion - -[build] -builder = cmake -subdir = double-conversion-3.1.4 diff --git a/build/fbcode_builder/manifests/double-conversion-python b/build/fbcode_builder/manifests/double-conversion-python deleted file mode 100644 index 0e1b376c9..000000000 --- a/build/fbcode_builder/manifests/double-conversion-python +++ /dev/null @@ -1,29 +0,0 @@ -[manifest] -name = double-conversion-python - -[download] -url = https://github.com/google/double-conversion/archive/v3.1.4.tar.gz -sha256 = 95004b65e43fefc6100f337a25da27bb99b9ef8d4071a36a33b5e83eb1f82021 - -[homebrew] -double-conversion - -[debs] -libdouble-conversion-dev - -[rpms] -double-conversion -double-conversion-devel - -[pps] -double-conversion - -[build] -builder = cmake -subdir = double-conversion-3.1.4 - -[build.not(os=linux)] -builder = nop - -[cmake.defines] -CMAKE_POSITION_INDEPENDENT_CODE=ON diff --git a/build/fbcode_builder/manifests/eden b/build/fbcode_builder/manifests/eden deleted file mode 100644 index 746ef7895..000000000 --- a/build/fbcode_builder/manifests/eden +++ /dev/null @@ -1,123 +0,0 @@ -[manifest] -name = eden -fbsource_path = fbcode/eden -shipit_project = eden -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebook/sapling.git - -[github.actions] -run_tests = off - -[sandcastle] -run_tests = off - -[build] -builder = cmake - -[dependencies] -blake3 -googletest -folly -fbthrift -fb303 -cpptoml -rocksdb -re2 -libgit2 -pexpect -python-psutil -python-toml -python-filelock -edencommon -rust-shed - -[dependencies.fbsource=on] -rust - -# macOS ships with sqlite3, and some of the core system -# frameworks require that that version be linked rather -# than the one we might build for ourselves here, so we -# skip building it on macos. -[dependencies.not(os=darwin)] -sqlite3 - -[dependencies.os=darwin] -osxfuse - -[dependencies.not(os=windows)] -# TODO: teach getdeps to compile curl on Windows. -# Enabling curl on Windows requires us to find a way to compile libcurl with -# msvc. -libcurl -# Added so that OSS doesn't see system "python" which is python 2 on darwin and some linux -python -# TODO: teach getdeps to compile lmdb on Windows. -lmdb - -[dependencies.test=on] -# sapling CLI is needed to run the tests -sapling - -[shipit.pathmap.fb=on] -# for internal builds that use getdeps -fbcode/fb303 = fb303 -fbcode/common/rust/shed = common/rust/shed -fbcode/thrift/lib/cpp = thrift/lib/cpp -fbcode/thrift/lib/cpp2 = thrift/lib/cpp2 -fbcode/thrift/lib/java = thrift/lib/java -fbcode/thrift/lib/py = thrift/lib/py -fbcode/thrift/lib/python = thrift/lib/python -fbcode/thrift/lib/rust = thrift/lib/rust - -[shipit.pathmap] -# Map hostcaps for now as eden C++ includes its .h. Rust-shed should install it -fbcode/common/rust/shed/hostcaps = common/rust/shed/hostcaps -fbcode/configerator/structs/scm/hg = configerator/structs/scm/hg -fbcode/eden/oss = . -fbcode/eden = eden -fbcode/tools/lfs = tools/lfs - -[shipit.pathmap.fb=off] -fbcode/eden/fs/public_autocargo = eden/fs -fbcode/eden/scm/public_autocargo = eden/scm -fbcode/common/rust/shed/hostcaps/public_cargo = common/rust/shed/hostcaps -fbcode/configerator/structs/scm/hg/public_autocargo = configerator/structs/scm/hg - -[shipit.strip] -^fbcode/eden/addons/.*$ -^fbcode/eden/fs/eden-config\.h$ -^fbcode/eden/fs/py/eden/config\.py$ -^fbcode/eden/hg-server/.*$ -^fbcode/eden/mononoke/(?!lfs_protocol) -^fbcode/eden/scm/build/.*$ -^fbcode/eden/scm/lib/third-party/rust/.*/Cargo.toml$ -^fbcode/eden/website/.*$ -^fbcode/eden/.*/\.cargo/.*$ -/Cargo\.lock$ -\.pyc$ - -[shipit.strip.fb=off] -^fbcode/common/rust/shed(?!/public_autocargo).*/Cargo\.toml$ -^fbcode/configerator/structs/scm/hg(?!/public_autocargo).*/Cargo\.toml$ -^fbcode/eden/fs(?!/public_autocargo).*/Cargo\.toml$ -^fbcode/eden/scm(?!/public_autocargo|/saplingnative).*/Cargo\.toml$ -^.*/facebook/.*$ -^.*/fb/.*$ - -[cmake.defines.all(fb=on,os=windows)] -ENABLE_GIT=OFF -INSTALL_PYTHON_LIB=ON - -[cmake.defines.all(not(fb=on),os=windows)] -ENABLE_GIT=OFF - -[cmake.defines.fbsource=on] -USE_CARGO_VENDOR=ON - -[cmake.defines.fb=on] -IS_FB_BUILD=ON - -[depends.environment] -EDEN_VERSION_OVERRIDE diff --git a/build/fbcode_builder/manifests/edencommon b/build/fbcode_builder/manifests/edencommon deleted file mode 100644 index e2c1b1167..000000000 --- a/build/fbcode_builder/manifests/edencommon +++ /dev/null @@ -1,32 +0,0 @@ -[manifest] -name = edencommon -fbsource_path = fbcode/eden/common -shipit_project = edencommon -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebookexperimental/edencommon.git - -[build] -builder = cmake - -[dependencies] -fbthrift -fb303 -fmt -folly -gflags -glog - -[cmake.defines.test=on] -BUILD_TESTS=ON - -[cmake.defines.test=off] -BUILD_TESTS=OFF - -[shipit.pathmap] -fbcode/eden/common = eden/common -fbcode/eden/common/oss = . - -[shipit.strip] -@README.facebook@ diff --git a/build/fbcode_builder/manifests/exprtk b/build/fbcode_builder/manifests/exprtk deleted file mode 100644 index c0dfc1afb..000000000 --- a/build/fbcode_builder/manifests/exprtk +++ /dev/null @@ -1,15 +0,0 @@ -[manifest] -name = exprtk - -[download] -url = https://github.com/ArashPartow/exprtk/archive/refs/tags/0.0.1.tar.gz -sha256 = fb72791c88ae3b3426e14fdad630027715682584daf56b973569718c56e33f28 - -[build.not(os=windows)] -builder = nop -subdir = exprtk-0.0.1 - -[install.files] -exprtk.hpp = exprtk.hpp - -[dependencies] diff --git a/build/fbcode_builder/manifests/fast_float b/build/fbcode_builder/manifests/fast_float deleted file mode 100644 index 531a1dd01..000000000 --- a/build/fbcode_builder/manifests/fast_float +++ /dev/null @@ -1,20 +0,0 @@ -[manifest] -name = fast_float - -[download] -url = https://github.com/fastfloat/fast_float/archive/refs/tags/v8.0.0.tar.gz -sha256 = f312f2dc34c61e665f4b132c0307d6f70ad9420185fa831911bc24408acf625d - -[build] -builder = cmake -subdir = fast_float-8.0.0 - -[cmake.defines] -FASTFLOAT_TEST = OFF -FASTFLOAT_SANITIZE = OFF - -[debs.not(all(distro=ubuntu,any(distro_vers="18.04",distro_vers="20.04",distro_vers="22.04",distro_vers="24.04")))] -libfast-float-dev - -[rpms.distro=fedora] -fast_float-devel diff --git a/build/fbcode_builder/manifests/fatal b/build/fbcode_builder/manifests/fatal deleted file mode 100644 index b516d765f..000000000 --- a/build/fbcode_builder/manifests/fatal +++ /dev/null @@ -1,24 +0,0 @@ -[manifest] -name = fatal -fbsource_path = fbcode/fatal -shipit_project = fatal - -[git] -repo_url = https://github.com/facebook/fatal.git - -[shipit.pathmap] -fbcode/fatal = fatal -fbcode/fatal/public_tld = . - -[build] -builder = nop -subdir = . - -[install.files] -fatal/portability.h = fatal/portability.h -fatal/preprocessor.h = fatal/preprocessor.h -fatal/container = fatal/container -fatal/functional = fatal/functional -fatal/math = fatal/math -fatal/string = fatal/string -fatal/type = fatal/type diff --git a/build/fbcode_builder/manifests/fb303 b/build/fbcode_builder/manifests/fb303 deleted file mode 100644 index cd34c085e..000000000 --- a/build/fbcode_builder/manifests/fb303 +++ /dev/null @@ -1,37 +0,0 @@ -[manifest] -name = fb303 -fbsource_path = fbcode/fb303 -shipit_project = fb303 -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebook/fb303.git - -[cargo] -cargo_config_file = source/fb303/thrift/.cargo/config.toml - -[crate.pathmap] -fb303_core = fb303/thrift/rust - -[build] -builder = cmake - -[dependencies] -folly -gflags -glog -fbthrift - -[cmake.defines.test=on] -BUILD_TESTS=ON - -[cmake.defines.test=off] -BUILD_TESTS=OFF - -[shipit.pathmap] -fbcode/fb303/github = . -fbcode/fb303/public_autocargo = fb303 -fbcode/fb303 = fb303 - -[shipit.strip] -^fbcode/fb303/(?!public_autocargo).+/Cargo\.toml$ diff --git a/build/fbcode_builder/manifests/fboss b/build/fbcode_builder/manifests/fboss deleted file mode 100644 index 6df183150..000000000 --- a/build/fbcode_builder/manifests/fboss +++ /dev/null @@ -1,63 +0,0 @@ -[manifest] -name = fboss -fbsource_path = fbcode/fboss -shipit_project = fboss -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebook/fboss.git - -[build.os=linux] -builder = cmake -# fboss files take a lot of RAM to compile. -job_weight_mib = 3072 - -[build.not(os=linux)] -builder = nop - -[dependencies] -folly -fb303 -wangle -fizz -mvfst -fmt -libsodium -googletest -zstd -fatal -fbthrift -iproute2 -libusb -libcurl -libnl -libsai -re2 -python -yaml-cpp -libyaml -CLI11 -exprtk -nlohmann-json -libgpiod -systemd -range-v3 -tabulate -gcc12 -python-pyyaml - -[shipit.pathmap] -fbcode/fboss/github = . -fbcode/fboss/common = common -fbcode/fboss = fboss - -[shipit.strip] -^fbcode/fboss/github/docs/.* -^fbcode/fboss/oss/.* -^fbcode/fboss/github/.github/.* -^fbcode/fboss/github/fboss-image/.* -^fbcode/fboss/github/.pre-commit-config.yaml -^fbcode/fboss/github/requirements-dev.txt - -[sandcastle] -run_tests = off diff --git a/build/fbcode_builder/manifests/fbthrift b/build/fbcode_builder/manifests/fbthrift deleted file mode 100644 index ebac705dc..000000000 --- a/build/fbcode_builder/manifests/fbthrift +++ /dev/null @@ -1,48 +0,0 @@ -[manifest] -name = fbthrift -fbsource_path = xplat/thrift -shipit_project = fbthrift -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebook/fbthrift.git - -[cargo] -cargo_config_file = source/thrift/lib/rust/.cargo/config.toml - -[crate.pathmap] -fbthrift = thrift/lib/rust - -[build] -builder = cmake -job_weight_mib = 2048 - -[cmake.defines.all(not(os=windows),test=on)] -enable_tests=ON - -[cmake.defines.any(os=windows,test=off)] -enable_tests=OFF - -[dependencies] -fizz -fmt -folly -googletest -libsodium -wangle -zstd -mvfst -xxhash -# Thrift also depends on openssl but since the latter requires a platform- -# specific configuration we rely on the folly manifest to provide this -# dependency to avoid duplication. - -[shipit.pathmap] -xplat/thrift/public_tld = . -xplat/thrift = thrift - -[shipit.strip] -^xplat/thrift/thrift-config\.h$ -^xplat/thrift/perf/canary.py$ -^xplat/thrift/perf/loadtest.py$ -^xplat/thrift/.castle/.* diff --git a/build/fbcode_builder/manifests/fbthrift-python b/build/fbcode_builder/manifests/fbthrift-python deleted file mode 100644 index 5609b8eef..000000000 --- a/build/fbcode_builder/manifests/fbthrift-python +++ /dev/null @@ -1,60 +0,0 @@ -[manifest] -name = fbthrift-python -fbsource_path = xplat/thrift -shipit_project = fbthrift -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebook/fbthrift.git - -[cargo] -cargo_config_file = source/thrift/lib/rust/.cargo/config.toml - -[crate.pathmap] -fbthrift = thrift/lib/rust - -[build] -builder = cmake -job_weight_mib = 2048 - -[build.not(os=linux)] -builder = nop - -[cmake.defines.all(not(os=windows),test=on)] -enable_tests=ON - -[cmake.defines.any(os=windows,test=off)] -enable_tests=OFF - -[cmake.defines.os=linux] -thrift_python=ON -enable_tests=ON - -[dependencies] -fizz-python -fmt-python -folly-python -googletest -libsodium -wangle-python -zstd-python -mvfst-python -xxhash -# Thrift also depends on openssl but since the latter requires a platform- -# specific configuration we rely on the folly manifest to provide this -# dependency to avoid duplication. - -[dependencies.os=linux] -libaio -libevent-python -proxygen-python - -[shipit.pathmap] -xplat/thrift/public_tld = . -xplat/thrift = thrift - -[shipit.strip] -^xplat/thrift/thrift-config\.h$ -^xplat/thrift/perf/canary.py$ -^xplat/thrift/perf/loadtest.py$ -^xplat/thrift/.castle/.* diff --git a/build/fbcode_builder/manifests/fizz b/build/fbcode_builder/manifests/fizz deleted file mode 100644 index 3709ff9d6..000000000 --- a/build/fbcode_builder/manifests/fizz +++ /dev/null @@ -1,38 +0,0 @@ -[manifest] -name = fizz -fbsource_path = fbcode/fizz -shipit_project = fizz -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebookincubator/fizz.git - -[build] -builder = cmake -subdir = fizz - -[cmake.defines] -BUILD_EXAMPLES = OFF - -[cmake.defines.test=on] -BUILD_TESTS = ON - -[cmake.defines.all(os=windows, test=on)] -BUILD_TESTS = OFF - -[cmake.defines.test=off] -BUILD_TESTS = OFF - -[dependencies] -folly -liboqs -libsodium -zlib -zstd - -[dependencies.all(test=on, not(os=windows))] -googletest - -[shipit.pathmap] -fbcode/fizz/public_tld = . -fbcode/fizz = fizz diff --git a/build/fbcode_builder/manifests/fizz-python b/build/fbcode_builder/manifests/fizz-python deleted file mode 100644 index 8ec678744..000000000 --- a/build/fbcode_builder/manifests/fizz-python +++ /dev/null @@ -1,45 +0,0 @@ -[manifest] -name = fizz-python -fbsource_path = fbcode/fizz -shipit_project = fizz -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebookincubator/fizz.git - -[build] -builder = cmake -subdir = fizz - -[build.not(os=linux)] -builder = nop - -[cmake.defines] -BUILD_EXAMPLES = OFF - -[cmake.defines.os=linux] -CMAKE_POSITION_INDEPENDENT_CODE = ON -BUILD_SHARED_LIBS = ON - -[cmake.defines.test=on] -BUILD_TESTS = ON - -[cmake.defines.all(os=windows, test=on)] -BUILD_TESTS = OFF - -[cmake.defines.test=off] -BUILD_TESTS = OFF - -[dependencies] -folly-python -liboqs -libsodium -zlib-python -zstd-python - -[dependencies.all(test=on, not(os=windows))] -googletest - -[shipit.pathmap] -fbcode/fizz/public_tld = . -fbcode/fizz = fizz diff --git a/build/fbcode_builder/manifests/fmt b/build/fbcode_builder/manifests/fmt deleted file mode 100644 index 9799d9f0a..000000000 --- a/build/fbcode_builder/manifests/fmt +++ /dev/null @@ -1,20 +0,0 @@ -[manifest] -name = fmt - -[download] -url = https://github.com/fmtlib/fmt/archive/refs/tags/12.1.0.tar.gz -sha256 = ea7de4299689e12b6dddd392f9896f08fb0777ac7168897a244a6d6085043fea - -[build] -builder = cmake -subdir = fmt-12.1.0 - -[cmake.defines] -FMT_TEST = OFF -FMT_DOC = OFF - -[homebrew] -fmt - -[rpms.distro=fedora] -fmt-devel diff --git a/build/fbcode_builder/manifests/fmt-python b/build/fbcode_builder/manifests/fmt-python deleted file mode 100644 index dfcd9e2b3..000000000 --- a/build/fbcode_builder/manifests/fmt-python +++ /dev/null @@ -1,26 +0,0 @@ -[manifest] -name = fmt-python - -[download] -url = https://github.com/fmtlib/fmt/archive/refs/tags/12.1.0.tar.gz -sha256 = ea7de4299689e12b6dddd392f9896f08fb0777ac7168897a244a6d6085043fea - -[build] -builder = cmake -subdir = fmt-12.1.0 - -[build.not(os=linux)] -builder = nop - -[cmake.defines] -FMT_TEST = OFF -FMT_DOC = OFF -# Build as shared library so Python extensions can find fmt symbols at runtime -# (fmt uses -fvisibility=hidden, so static linking leaves symbols unexported) -BUILD_SHARED_LIBS = ON - -[homebrew] -fmt - -[rpms.distro=fedora] -fmt-devel diff --git a/build/fbcode_builder/manifests/folly b/build/fbcode_builder/manifests/folly deleted file mode 100644 index c9ac2cf37..000000000 --- a/build/fbcode_builder/manifests/folly +++ /dev/null @@ -1,80 +0,0 @@ -[manifest] -name = folly -fbsource_path = fbcode/folly -shipit_project = folly -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebook/folly.git - -[build] -builder = cmake -job_weight_mib = 1024 - -[dependencies] -gflags -glog -googletest -boost -libdwarf -libevent -libsodium -double-conversion -fast_float -fmt -lz4 -snappy -zstd -# no openssl or zlib in the linux case, why? -# these are usually installed on the system -# and are the easiest system deps to pull in. -# In the future we want to be able to express -# that a system dep is sufficient in the manifest -# for eg: openssl and zlib, but for now we don't -# have it. - -# macOS doesn't expose the openssl api so we need -# to build our own. -[dependencies.os=darwin] -openssl - -# Windows has neither openssl nor zlib, so we get -# to provide both -[dependencies.os=windows] -openssl -zlib - -[dependencies.os=linux] -libaio -libiberty -libunwind - -# xz depends on autoconf which does not build on -# Windows -[dependencies.not(os=windows)] -xz - -[shipit.pathmap] -fbcode/folly/public_tld = . -fbcode/folly = folly - -[shipit.strip] -^fbcode/folly/folly-config\.h$ -^fbcode/folly/public_tld/build/facebook_.* - -[cmake.defines] -BUILD_SHARED_LIBS=OFF - -[cmake.defines.not(os=windows)] -BOOST_LINK_STATIC=ON - -[cmake.defines.os=freebsd] -LIBDWARF_FOUND=NO - -[cmake.defines.test=on] -BUILD_TESTS=ON -BUILD_BENCHMARKS=OFF - -[cmake.defines.test=off] -BUILD_TESTS=OFF -BUILD_BENCHMARKS=OFF diff --git a/build/fbcode_builder/manifests/folly-python b/build/fbcode_builder/manifests/folly-python deleted file mode 100644 index 02c9193e8..000000000 --- a/build/fbcode_builder/manifests/folly-python +++ /dev/null @@ -1,84 +0,0 @@ -[manifest] -name = folly-python -fbsource_path = fbcode/folly -shipit_project = folly -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebook/folly.git - -[build] -builder = cmake -job_weight_mib = 1024 - -[build.not(os=linux)] -builder = nop - -[dependencies] -gflags -glog -googletest -boost-python -libdwarf-python -libevent-python -libsodium -double-conversion-python -fast_float -fmt-python -lz4-python -snappy -zstd-python -# no openssl or zlib in the linux case, why? -# these are usually installed on the system -# and are the easiest system deps to pull in. -# In the future we want to be able to express -# that a system dep is sufficient in the manifest -# for eg: openssl and zlib, but for now we don't -# have it. - -# macOS doesn't expose the openssl api so we need -# to build our own. -[dependencies.os=darwin] -openssl - -# Windows has neither openssl nor zlib, so we get -# to provide both -[dependencies.os=windows] -openssl -zlib - -[dependencies.os=linux] -libaio -libiberty-python -libunwind - -# xz depends on autoconf which does not build on -# Windows -[dependencies.not(os=windows)] -xz - -[shipit.pathmap] -fbcode/folly/public_tld = . -fbcode/folly = folly - -[shipit.strip] -^fbcode/folly/folly-config\.h$ -^fbcode/folly/public_tld/build/facebook_.* - -[cmake.defines.os=linux] -PYTHON_EXTENSIONS=ON -BUILD_SHARED_LIBS=ON - -[cmake.defines.not(os=windows)] -BOOST_LINK_STATIC=ON - -[cmake.defines.os=freebsd] -LIBDWARF_FOUND=NO - -[cmake.defines.test=on] -BUILD_TESTS=ON -BUILD_BENCHMARKS=OFF - -[cmake.defines.test=off] -BUILD_TESTS=OFF -BUILD_BENCHMARKS=OFF diff --git a/build/fbcode_builder/manifests/gcc12 b/build/fbcode_builder/manifests/gcc12 deleted file mode 100644 index 039a55269..000000000 --- a/build/fbcode_builder/manifests/gcc12 +++ /dev/null @@ -1,5 +0,0 @@ -[manifest] -name = gcc12 - -[rpms.all(distro=centos_stream,distro_vers=9)] -gcc-toolset-12 diff --git a/build/fbcode_builder/manifests/gcc14 b/build/fbcode_builder/manifests/gcc14 deleted file mode 100644 index 45a9c238e..000000000 --- a/build/fbcode_builder/manifests/gcc14 +++ /dev/null @@ -1,5 +0,0 @@ -[manifest] -name = gcc14 - -[rpms.all(distro=centos_stream,distro_vers=9)] -gcc-toolset-14 diff --git a/build/fbcode_builder/manifests/gflags b/build/fbcode_builder/manifests/gflags deleted file mode 100644 index 47c01c204..000000000 --- a/build/fbcode_builder/manifests/gflags +++ /dev/null @@ -1,25 +0,0 @@ -[manifest] -name = gflags - -[download] -url = https://github.com/gflags/gflags/archive/v2.2.2.tar.gz -sha256 = 34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf - -[build] -builder = cmake -subdir = gflags-2.2.2 - -[cmake.defines] -BUILD_SHARED_LIBS = ON -BUILD_STATIC_LIBS = ON -#BUILD_gflags_nothreads_LIB = OFF -BUILD_gflags_LIB = ON - -[homebrew] -gflags - -[debs] -libgflags-dev - -[rpms.distro=fedora] -gflags-devel diff --git a/build/fbcode_builder/manifests/ghc b/build/fbcode_builder/manifests/ghc deleted file mode 100644 index 2da8f5ffd..000000000 --- a/build/fbcode_builder/manifests/ghc +++ /dev/null @@ -1,15 +0,0 @@ -[manifest] -name = ghc - -[download.os=linux] -url = https://downloads.haskell.org/~ghc/9.2.8/ghc-9.2.8-x86_64-fedora27-linux.tar.xz -sha256 = 845f63cd365317bb764d81025554a2527dbe315d6fa268c9859e21b911bf2d3c - -[build] -builder = autoconf -subdir = ghc-9.2.8 -build_in_src_dir = true -only_install = true - -[make.install_args] -install diff --git a/build/fbcode_builder/manifests/git-lfs b/build/fbcode_builder/manifests/git-lfs deleted file mode 100644 index 19b24e247..000000000 --- a/build/fbcode_builder/manifests/git-lfs +++ /dev/null @@ -1,15 +0,0 @@ -[manifest] -name = git-lfs - -[rpms] -git-lfs - -[debs] -git-lfs - -[homebrew] -git-lfs - -# only used from system packages currently -[build] -builder = nop diff --git a/build/fbcode_builder/manifests/glean b/build/fbcode_builder/manifests/glean deleted file mode 100644 index cddf8b7ba..000000000 --- a/build/fbcode_builder/manifests/glean +++ /dev/null @@ -1,48 +0,0 @@ -[manifest] -name = glean -fbsource_path = fbcode/glean -shipit_project = facebookincubator/Glean -use_shipit = true - -[shipit.pathmap] -# These are only used by target determinator to trigger builds, the -# real path mappings are in the ShipIt config. -fbcode/glean = glean -fbcode/common/hs = hsthrift - -[subprojects] -hsthrift = hsthrift - -[dependencies] -cabal -ghc -gflags -glog -folly -rocksdb -xxhash -llvm -clang -re2 - -[build] -builder = make - -[make.build_args] -setup-folly -setup-folly-version -cabal-update -all -glean-hie -glass -glean-clang -EXTRA_GHC_OPTS=-j4 +RTS -A32m -n4m -RTS -CABAL_CONFIG_FLAGS=-f-hack-tests -f-typescript-tests -f-python-tests -f-dotnet-tests -f-go-tests -f-rust-tests -f-java-lsif-tests -f-flow-tests -f-bundled-folly - -[make.install_args] -install - -[make.test_args] -test -EXTRA_GHC_OPTS=-j4 +RTS -A32m -n4m -RTS -CABAL_CONFIG_FLAGS=-f-hack-tests -f-typescript-tests -f-python-tests -f-dotnet-tests -f-go-tests -f-rust-tests -f-java-lsif-tests -f-flow-tests -f-bundled-folly diff --git a/build/fbcode_builder/manifests/glog b/build/fbcode_builder/manifests/glog deleted file mode 100644 index 2649eaad1..000000000 --- a/build/fbcode_builder/manifests/glog +++ /dev/null @@ -1,33 +0,0 @@ -[manifest] -name = glog - -[download] -url = https://github.com/google/glog/archive/v0.5.0.tar.gz -sha256 = eede71f28371bf39aa69b45de23b329d37214016e2055269b3b5e7cfd40b59f5 - -[build] -builder = cmake -subdir = glog-0.5.0 - -[dependencies] -gflags - -[cmake.defines] -BUILD_SHARED_LIBS=ON -BUILD_TESTING=NO -WITH_PKGCONFIG=ON - -[cmake.defines.os=freebsd] -HAVE_TR1_UNORDERED_MAP=OFF -HAVE_TR1_UNORDERED_SET=OFF - -[homebrew] -glog - -# on ubuntu glog brings in liblzma-dev, which in turn breaks watchman tests -[debs.not(distro=ubuntu)] -libgoogle-glog-dev - -[rpms.distro=fedora] -glog-devel - diff --git a/build/fbcode_builder/manifests/googletest b/build/fbcode_builder/manifests/googletest deleted file mode 100644 index 101175874..000000000 --- a/build/fbcode_builder/manifests/googletest +++ /dev/null @@ -1,30 +0,0 @@ -[manifest] -name = googletest - -[download] -url = https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz -sha256 = 81964fe578e9bd7c94dfdb09c8e4d6e6759e19967e397dbea48d1c10e45d0df2 - -[build] -builder = cmake -subdir = googletest-release-1.12.1 - -[cmake.defines] -# Everything else defaults to the shared runtime, so tell gtest that -# it should not use its choice of the static runtime -gtest_force_shared_crt=ON - -[cmake.defines.os=windows] -BUILD_SHARED_LIBS=ON - -[homebrew] -googletest - -# packaged googletest is too old -[debs.not(all(distro=ubuntu,any(distro_vers="18.04",distro_vers="20.04",distro_vers="22.04")))] -libgtest-dev -libgmock-dev - -[rpms.distro=fedora] -gmock-devel -gtest-devel diff --git a/build/fbcode_builder/manifests/gperf b/build/fbcode_builder/manifests/gperf deleted file mode 100644 index 3c45fe173..000000000 --- a/build/fbcode_builder/manifests/gperf +++ /dev/null @@ -1,13 +0,0 @@ -[manifest] -name = gperf - -[download] -url = https://ftpmirror.gnu.org/gnu/gperf/gperf-3.1.tar.gz -sha256 = 588546b945bba4b70b6a3a616e80b4ab466e3f33024a352fc2198112cdbb3ae2 - -[build.not(os=windows)] -builder = autoconf -subdir = gperf-3.1 - -[build.os=windows] -builder = nop diff --git a/build/fbcode_builder/manifests/hexdump b/build/fbcode_builder/manifests/hexdump deleted file mode 100644 index e80674f14..000000000 --- a/build/fbcode_builder/manifests/hexdump +++ /dev/null @@ -1,12 +0,0 @@ -[manifest] -name = hexdump - -[rpms] -util-linux - -[debs] -bsdmainutils - -# only used from system packages currently -[build] -builder = nop diff --git a/build/fbcode_builder/manifests/hsthrift b/build/fbcode_builder/manifests/hsthrift deleted file mode 100644 index fbab594bc..000000000 --- a/build/fbcode_builder/manifests/hsthrift +++ /dev/null @@ -1,36 +0,0 @@ -[manifest] -name = hsthrift -fbsource_path = fbcode/common/hs -shipit_project = facebookincubator/hsthrift -use_shipit = true - -[shipit.pathmap] -# These are only used by target determinator to trigger builds, the -# real path mappings are in the ShipIt config. -fbcode/common/hs = . - -[dependencies] -cabal -ghc -gflags -glog -folly -fbthrift -wangle -fizz -boost - -[build] -builder = make - -[make.build_args] -setup-folly -setup-meta -cabal-update -all - -[make.install_args] -install - -[make.test_args] -test diff --git a/build/fbcode_builder/manifests/iproute2 b/build/fbcode_builder/manifests/iproute2 deleted file mode 100644 index f7f3e766a..000000000 --- a/build/fbcode_builder/manifests/iproute2 +++ /dev/null @@ -1,14 +0,0 @@ -[manifest] -name = iproute2 - -[download] -url = https://mirrors.edge.kernel.org/pub/linux/utils/net/iproute2/iproute2-4.12.0.tar.gz -sha256 = 46612a1e2d01bb31932557bccdb1b8618cae9a439dfffc08ef35ed8e197f14ce - -[build.os=linux] -builder = iproute2 -subdir = iproute2-4.12.0 -patchfile = iproute2_oss.patch - -[build.not(os=linux)] -builder = nop diff --git a/build/fbcode_builder/manifests/jom b/build/fbcode_builder/manifests/jom deleted file mode 100644 index effecab67..000000000 --- a/build/fbcode_builder/manifests/jom +++ /dev/null @@ -1,15 +0,0 @@ -# jom is compatible with MSVC nmake, but adds the /j argment which -# speeds up openssl build a lot -[manifest] -name = jom - -# see https://download.qt.io/official_releases/jom/changelog.txt for latest version -[download.os=windows] -url = https://download.qt.io/official_releases/jom/jom_1_1_4.zip -sha256 = d533c1ef49214229681e90196ed2094691e8c4a0a0bef0b2c901debcb562682b - -[build.os=windows] -builder = nop - -[install.files.os=windows] -. = bin diff --git a/build/fbcode_builder/manifests/jq b/build/fbcode_builder/manifests/jq deleted file mode 100644 index 354854b2c..000000000 --- a/build/fbcode_builder/manifests/jq +++ /dev/null @@ -1,25 +0,0 @@ -[manifest] -name = jq - -[rpms.distro=fedora] -jq - -[homebrew] -jq - -[download.not(os=windows)] -# we use jq-1.7+ to get fix for number truncation https://github.com/jqlang/jq/pull/1752 -url = https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-1.7.1.tar.gz -sha256 = 478c9ca129fd2e3443fe27314b455e211e0d8c60bc8ff7df703873deeee580c2 - -[build.not(os=windows)] -builder = autoconf -subdir = jq-1.7.1 - -[build.os=windows] -builder = nop - -[autoconf.args] -# This argument turns off some developers tool and it is recommended in jq's -# README ---disable-maintainer-mode diff --git a/build/fbcode_builder/manifests/katran b/build/fbcode_builder/manifests/katran deleted file mode 100644 index c4f2c74f4..000000000 --- a/build/fbcode_builder/manifests/katran +++ /dev/null @@ -1,41 +0,0 @@ -[manifest] -name = katran -fbsource_path = fbcode/katran -shipit_project = katran -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebookincubator/katran.git - -[build.not(os=linux)] -builder = nop - -[build.os=linux] -builder = cmake -subdir = . - -[cmake.defines.test=on] -BUILD_TESTS=ON - -[cmake.defines.test=off] -BUILD_TESTS=OFF - -[dependencies] -folly -fizz -libbpf -libmnl -zlib -googletest -fmt - -[debs] -libssl-dev - -[shipit.pathmap] -fbcode/katran/public_root = . -fbcode/katran = katran - -[shipit.strip] -^fbcode/katran/facebook -^fbcode/katran/OSS_SYNC diff --git a/build/fbcode_builder/manifests/libaio b/build/fbcode_builder/manifests/libaio deleted file mode 100644 index e35991923..000000000 --- a/build/fbcode_builder/manifests/libaio +++ /dev/null @@ -1,8 +0,0 @@ -[manifest] -name = libaio - -[debs] -libaio-dev - -[rpms.distro=centos_stream] -libaio-devel diff --git a/build/fbcode_builder/manifests/libbpf b/build/fbcode_builder/manifests/libbpf deleted file mode 100644 index 4d0ed71be..000000000 --- a/build/fbcode_builder/manifests/libbpf +++ /dev/null @@ -1,26 +0,0 @@ -[manifest] -name = libbpf - -[download] -url = https://github.com/libbpf/libbpf/archive/refs/tags/v1.6.2.tar.gz -sha256 = 16f31349c70764cba8e0fad3725cc9f52f6cf952554326aa0229daaa21ef4fbd - -# BPF only builds on linux, so make it a NOP on other platforms -[build.not(os=linux)] -builder = nop - -[build.os=linux] -builder = make -subdir = libbpf-1.6.2/src - -[make.build_args] -BUILD_STATIC_ONLY=y - -# libbpf-0.3 requires uapi headers >= 5.8 -[make.install_args] -install -install_uapi_headers -BUILD_STATIC_ONLY=y - -[dependencies] -libelf diff --git a/build/fbcode_builder/manifests/libcurl b/build/fbcode_builder/manifests/libcurl deleted file mode 100644 index cc2ac3992..000000000 --- a/build/fbcode_builder/manifests/libcurl +++ /dev/null @@ -1,42 +0,0 @@ -[manifest] -name = libcurl - -[rpms] -libcurl-devel -libcurl-minimal - -[debs] -libcurl4-openssl-dev - -[pps] -libcurl-gnutls - -[download] -url = https://curl.haxx.se/download/curl-7.65.1.tar.gz -sha256 = 821aeb78421375f70e55381c9ad2474bf279fc454b791b7e95fc83562951c690 - -[dependencies] -nghttp2 - -# We use system OpenSSL on Linux (see folly's manifest for details) -[dependencies.not(os=linux)] -openssl - -[build.not(os=windows)] -builder = autoconf -subdir = curl-7.65.1 - -[autoconf.args] -# fboss (which added the libcurl dep) doesn't need ldap so it is disabled here. -# if someone in the future wants to add ldap for something else, it won't hurt -# fboss. However, that would require adding an ldap manifest. -# -# For the same reason, we disable libssh2 and libidn2 which aren't really used -# but would require adding manifests if we don't disable them. ---disable-ldap ---without-libssh2 ---without-libidn2 - -[build.os=windows] -builder = cmake -subdir = curl-7.65.1 diff --git a/build/fbcode_builder/manifests/libdwarf b/build/fbcode_builder/manifests/libdwarf deleted file mode 100644 index e93ba16bc..000000000 --- a/build/fbcode_builder/manifests/libdwarf +++ /dev/null @@ -1,20 +0,0 @@ -[manifest] -name = libdwarf - -[rpms] -libdwarf-devel -libdwarf - -[debs] -libdwarf-dev - -[homebrew] -dwarfutils - -[download] -url = https://www.prevanders.net/libdwarf-0.9.2.tar.xz -sha256 = 22b66d06831a76f6a062126cdcad3fcc58540b89a1acb23c99f8861f50999ec3 - -[build] -builder = cmake -subdir = libdwarf-0.9.2 diff --git a/build/fbcode_builder/manifests/libdwarf-python b/build/fbcode_builder/manifests/libdwarf-python deleted file mode 100644 index f3cc29910..000000000 --- a/build/fbcode_builder/manifests/libdwarf-python +++ /dev/null @@ -1,26 +0,0 @@ -[manifest] -name = libdwarf-python - -[rpms] -libdwarf-devel -libdwarf - -[debs] -libdwarf-dev - -[homebrew] -dwarfutils - -[download] -url = https://www.prevanders.net/libdwarf-0.9.2.tar.xz -sha256 = 22b66d06831a76f6a062126cdcad3fcc58540b89a1acb23c99f8861f50999ec3 - -[build] -builder = cmake -subdir = libdwarf-0.9.2 - -[build.not(os=linux)] -builder = nop - -[cmake.defines] -CMAKE_POSITION_INDEPENDENT_CODE=ON diff --git a/build/fbcode_builder/manifests/libelf b/build/fbcode_builder/manifests/libelf deleted file mode 100644 index c277ccd2e..000000000 --- a/build/fbcode_builder/manifests/libelf +++ /dev/null @@ -1,23 +0,0 @@ -[manifest] -name = libelf - -[rpms] -elfutils-libelf-devel-static - -[debs] -libelf-dev - -[pps] -libelf - -[download] -url = https://sourceware.org/elfutils/ftp/0.193/elfutils-0.193.tar.bz2 -sha256 = 7857f44b624f4d8d421df851aaae7b1402cfe6bcdd2d8049f15fc07d3dde7635 - -# libelf only makes sense on linux, so make it a NOP on other platforms -[build.not(os=linux)] -builder = nop - -[build.os=linux] -builder = autoconf -subdir = elfutils-0.193 diff --git a/build/fbcode_builder/manifests/libevent b/build/fbcode_builder/manifests/libevent deleted file mode 100644 index 91a2af90c..000000000 --- a/build/fbcode_builder/manifests/libevent +++ /dev/null @@ -1,41 +0,0 @@ -[manifest] -name = libevent - -[debs] -libevent-dev - -[homebrew] -libevent - -[rpms] -libevent-devel - -[pps] -libevent - -# Note that the CMakeLists.txt file is present only in -# git repo and not in the release tarball, so take care -# to use the github generated source tarball rather than -# the explicitly uploaded source tarball -[download] -url = https://github.com/libevent/libevent/releases/download/release-2.1.12-stable/libevent-2.1.12-stable.tar.gz -sha256 = 92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb - -[build] -builder = cmake -subdir = libevent-2.1.12-stable - -[cmake.defines] -EVENT__DISABLE_TESTS = ON -EVENT__DISABLE_BENCHMARK = ON -EVENT__DISABLE_SAMPLES = ON -EVENT__DISABLE_REGRESS = ON - -[cmake.defines.shared_libs=on] -EVENT__BUILD_SHARED_LIBRARIES = ON - -[cmake.defines.os=windows] -EVENT__LIBRARY_TYPE = STATIC - -[dependencies.not(any(os=linux, os=freebsd))] -openssl diff --git a/build/fbcode_builder/manifests/libevent-python b/build/fbcode_builder/manifests/libevent-python deleted file mode 100644 index 9f6fe4cf4..000000000 --- a/build/fbcode_builder/manifests/libevent-python +++ /dev/null @@ -1,43 +0,0 @@ -[manifest] -name = libevent-python - -# NOTE: System packages (debs, rpms) removed because they don't include -# LibeventConfig.cmake which is required by find_package(Libevent REQUIRED CONFIG). -# Building from source ensures CMake config files are present. - -[homebrew] -libevent - -[pps] -libevent - -# Note that the CMakeLists.txt file is present only in -# git repo and not in the release tarball, so take care -# to use the github generated source tarball rather than -# the explicitly uploaded source tarball -[download] -url = https://github.com/libevent/libevent/releases/download/release-2.1.12-stable/libevent-2.1.12-stable.tar.gz -sha256 = 92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb - -[build] -builder = cmake -subdir = libevent-2.1.12-stable - -[build.not(os=linux)] -builder = nop - -[cmake.defines] -EVENT__DISABLE_TESTS = ON -EVENT__DISABLE_BENCHMARK = ON -EVENT__DISABLE_SAMPLES = ON -EVENT__DISABLE_REGRESS = ON -CMAKE_POSITION_INDEPENDENT_CODE=ON - -[cmake.defines.shared_libs=on] -EVENT__BUILD_SHARED_LIBRARIES = ON - -[cmake.defines.os=windows] -EVENT__LIBRARY_TYPE = STATIC - -[dependencies.not(any(os=linux, os=freebsd))] -openssl diff --git a/build/fbcode_builder/manifests/libffi b/build/fbcode_builder/manifests/libffi deleted file mode 100644 index b520358fd..000000000 --- a/build/fbcode_builder/manifests/libffi +++ /dev/null @@ -1,23 +0,0 @@ -[manifest] -name = libffi - -[debs] -libffi-dev - -[homebrew] -libffi - -[rpms] -libffi-devel -libffi - -[pps] -libffi - -[download] -url = https://github.com/libffi/libffi/releases/download/v3.4.2/libffi-3.4.2.tar.gz -sha256 = 540fb721619a6aba3bdeef7d940d8e9e0e6d2c193595bc243241b77ff9e93620 - -[build] -builder = autoconf -subdir = libffi-3.4.2 diff --git a/build/fbcode_builder/manifests/libgit2 b/build/fbcode_builder/manifests/libgit2 deleted file mode 100644 index 42bbfca92..000000000 --- a/build/fbcode_builder/manifests/libgit2 +++ /dev/null @@ -1,33 +0,0 @@ -[manifest] -name = libgit2 - -[homebrew] -libgit2 - -[rpms] -libgit2-devel - -[pps] -libgit2 - -# Ubuntu 18.04 libgit2 has clash with libcurl4-openssl-dev as it depends on -# libcurl4-gnutls-dev. Should be ok from 20.04 again -# There is a description at https://github.com/r-hub/sysreqsdb/issues/77 -[debs.not(all(distro=ubuntu,distro_vers="18.04"))] -libgit2-dev - -[download] -url = https://github.com/libgit2/libgit2/archive/v0.28.1.tar.gz -sha256 = 0ca11048795b0d6338f2e57717370208c2c97ad66c6d5eac0c97a8827d13936b - -[build] -builder = cmake -subdir = libgit2-0.28.1 - -[cmake.defines] -# Could turn this on if we also wanted to add a manifest for libssh2 -USE_SSH = OFF -BUILD_CLAR = OFF -# Have to build shared to work around annoying problems with cmake -# mis-parsing the frameworks required to link this on macos :-/ -BUILD_SHARED_LIBS = ON diff --git a/build/fbcode_builder/manifests/libgpiod b/build/fbcode_builder/manifests/libgpiod deleted file mode 100644 index 24f7dd675..000000000 --- a/build/fbcode_builder/manifests/libgpiod +++ /dev/null @@ -1,10 +0,0 @@ -[manifest] -name = libgpiod - -[download] -url = https://cdn.kernel.org/pub/software/libs/libgpiod/libgpiod-1.6.tar.xz -sha256 = 62908023d59e8cbb9137ddd14deec50ced862d8f9b8749f288d3dbe7967151ef - -[build] -builder = autoconf -subdir = libgpiod-1.6 diff --git a/build/fbcode_builder/manifests/libiberty b/build/fbcode_builder/manifests/libiberty deleted file mode 100644 index a8bba1e73..000000000 --- a/build/fbcode_builder/manifests/libiberty +++ /dev/null @@ -1,29 +0,0 @@ -[manifest] -name = libiberty - -[rpms] -binutils-devel -binutils - -[debs.not(all(distro=ubuntu,distro_vers="24.04"))] -binutils-dev - -[debs.all(distro=ubuntu,distro_vers="24.04")] -binutils-x86-64-linux-gnu - -[download] -url = https://ftpmirror.gnu.org/gnu/binutils/binutils-2.43.tar.xz -sha256 = b53606f443ac8f01d1d5fc9c39497f2af322d99e14cea5c0b4b124d630379365 - -[dependencies] -zlib - -[build] -builder = autoconf -subdir = binutils-2.43/libiberty -patchfile = libiberty_install_pic_lib.patch - -# only build the parts needed for demangling -# as we still want to use system linker and assembler etc -[autoconf.args] ---enable-install-libiberty diff --git a/build/fbcode_builder/manifests/libiberty-python b/build/fbcode_builder/manifests/libiberty-python deleted file mode 100644 index 0afb4c0ae..000000000 --- a/build/fbcode_builder/manifests/libiberty-python +++ /dev/null @@ -1,32 +0,0 @@ -[manifest] -name = libiberty-python - -[rpms] -binutils-devel -binutils - -[debs.not(all(distro=ubuntu,distro_vers="24.04"))] -binutils-dev - -[debs.all(distro=ubuntu,distro_vers="24.04")] -binutils-x86-64-linux-gnu - -[download] -url = https://ftpmirror.gnu.org/gnu/binutils/binutils-2.43.tar.xz -sha256 = b53606f443ac8f01d1d5fc9c39497f2af322d99e14cea5c0b4b124d630379365 - -[dependencies] -zlib-python - -[build] -builder = autoconf -subdir = binutils-2.43/libiberty -patchfile = libiberty_install_pic_lib.patch - -[build.not(os=linux)] -builder = nop - -# only build the parts needed for demangling -# as we still want to use system linker and assembler etc -[autoconf.args] ---enable-install-libiberty diff --git a/build/fbcode_builder/manifests/libibverbs b/build/fbcode_builder/manifests/libibverbs deleted file mode 100644 index 6239bd0f0..000000000 --- a/build/fbcode_builder/manifests/libibverbs +++ /dev/null @@ -1,28 +0,0 @@ -[manifest] -name = libibverbs - -[debs] -libibverbs-dev -rdma-core - -[rpms] -libibverbs -rdma-core-devel - -[download] -url = https://github.com/linux-rdma/rdma-core/releases/download/v60.0/rdma-core-60.0.tar.gz -sha256 = 9b1b892e4eaaaa5dfbade07a290fbf5079e39117724fa1ef80d0ad78839328de - -[build] -builder = cmake -subdir = rdma-core-60.0 - -[dependencies] -libnl - -[cmake.defines] -NO_MAN_PAGES=1 -NO_PYVERBS=1 -ENABLE_RESOLVE_NEIGH=0 -# Use absolute short path for runtime dir to avoid Unix socket path length limit (108 chars) -CMAKE_INSTALL_RUNDIR=/tmp/ibacm diff --git a/build/fbcode_builder/manifests/libmnl b/build/fbcode_builder/manifests/libmnl deleted file mode 100644 index 99861239d..000000000 --- a/build/fbcode_builder/manifests/libmnl +++ /dev/null @@ -1,24 +0,0 @@ -[manifest] -name = libmnl - -[rpms] -libmnl-devel - -# all centos 8 distros are missing this, -# but its in fedora so may be back in a later version -[rpms.not(all(any(distro=centos_stream,distro=centos),distro_vers=8))] -libmnl-static - -[debs] -libmnl-dev - -[pps] -libmnl - -[download] -url = https://www.netfilter.org/pub/libmnl/libmnl-1.0.4.tar.bz2 -sha256 = 171f89699f286a5854b72b91d06e8f8e3683064c5901fb09d954a9ab6f551f81 - -[build.os=linux] -builder = autoconf -subdir = libmnl-1.0.4 diff --git a/build/fbcode_builder/manifests/libnl b/build/fbcode_builder/manifests/libnl deleted file mode 100644 index 003623343..000000000 --- a/build/fbcode_builder/manifests/libnl +++ /dev/null @@ -1,21 +0,0 @@ -[manifest] -name = libnl - -[rpms] -libnl3-devel -libnl3 - -[debs] -libnl-3-dev -libnl-route-3-dev - -[pps] -libnl - -[download] -url = https://github.com/thom311/libnl/releases/download/libnl3_2_25/libnl-3.2.25.tar.gz -sha256 = 8beb7590674957b931de6b7f81c530b85dc7c1ad8fbda015398bc1e8d1ce8ec5 - -[build.os=linux] -builder = autoconf -subdir = libnl-3.2.25 diff --git a/build/fbcode_builder/manifests/liboqs b/build/fbcode_builder/manifests/liboqs deleted file mode 100644 index 74dcfd5b3..000000000 --- a/build/fbcode_builder/manifests/liboqs +++ /dev/null @@ -1,16 +0,0 @@ -[manifest] -name = liboqs - -[download] -url = https://github.com/open-quantum-safe/liboqs/archive/refs/tags/0.12.0.tar.gz -sha256 = df999915204eb1eba311d89e83d1edd3a514d5a07374745d6a9e5b2dd0d59c08 - -[build] -builder = cmake -subdir = liboqs-0.12.0 - -[cmake.defines] -OQS_MINIMAL_BUILD = KEM_kyber_512;KEM_kyber_768;KEM_kyber_1024;KEM_ml_kem_512;KEM_ml_kem_768;KEM_ml_kem_1024 - -[dependencies] -openssl diff --git a/build/fbcode_builder/manifests/libsai b/build/fbcode_builder/manifests/libsai deleted file mode 100644 index 31b9a5bc6..000000000 --- a/build/fbcode_builder/manifests/libsai +++ /dev/null @@ -1,14 +0,0 @@ -[manifest] -name = libsai - -[download] -url = https://github.com/opencomputeproject/SAI/archive/v1.16.3.tar.gz -sha256 = 5c89cdb6b2e4f1b42ced6b78d43d06d22434ddbf423cdc551f7c2001f12e63d9 - -[build] -builder = nop -subdir = SAI-1.16.3 - -[install.files] -inc = include -experimental = experimental diff --git a/build/fbcode_builder/manifests/libsodium b/build/fbcode_builder/manifests/libsodium deleted file mode 100644 index 2cdeb8c78..000000000 --- a/build/fbcode_builder/manifests/libsodium +++ /dev/null @@ -1,39 +0,0 @@ -[manifest] -name = libsodium - -[debs] -libsodium-dev - -[homebrew] -libsodium - -[rpms] -libsodium-devel -libsodium-static - -[pps] -libsodium - -[download.not(os=windows)] -url = https://github.com/jedisct1/libsodium/releases/download/1.0.20-RELEASE/libsodium-1.0.20.tar.gz -sha256 = ebb65ef6ca439333c2bb41a0c1990587288da07f6c7fd07cb3a18cc18d30ce19 - -[build.not(os=windows)] -builder = autoconf -subdir = libsodium-1.0.20 - -[download.os=windows] -url = https://github.com/jedisct1/libsodium/releases/download/1.0.20-RELEASE/libsodium-1.0.20-msvc.zip -sha256 = 2ff97f9e3f5b341bdc808e698057bea1ae454f99e29ff6f9b62e14d0eb1b1baa - -[build.os=windows] -builder = nop - -[install.files.os=windows] -libsodium/x64/Release/v143/dynamic/libsodium.dll = bin/libsodium.dll -libsodium/x64/Release/v143/dynamic/libsodium.lib = lib/libsodium.lib -libsodium/x64/Release/v143/dynamic/libsodium.exp = lib/libsodium.exp -libsodium/x64/Release/v143/dynamic/libsodium.pdb = lib/libsodium.pdb -libsodium/include = include - -[autoconf.args] diff --git a/build/fbcode_builder/manifests/libtool b/build/fbcode_builder/manifests/libtool deleted file mode 100644 index 0630009d6..000000000 --- a/build/fbcode_builder/manifests/libtool +++ /dev/null @@ -1,28 +0,0 @@ -[manifest] -name = libtool - -[homebrew] -libtool - -[rpms] -libtool - -[debs] -libtool - -[pps] -libtool - -[download] -url = https://ftpmirror.gnu.org/gnu/libtool/libtool-2.4.6.tar.gz -sha256 = e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3 - -[build] -builder = autoconf -subdir = libtool-2.4.6 - -[dependencies] -automake - -[autoconf.args] ---enable-ltdl-install diff --git a/build/fbcode_builder/manifests/libunwind b/build/fbcode_builder/manifests/libunwind deleted file mode 100644 index 560edcd79..000000000 --- a/build/fbcode_builder/manifests/libunwind +++ /dev/null @@ -1,20 +0,0 @@ -[manifest] -name = libunwind - -[rpms] -libunwind-devel -libunwind - -# on ubuntu this brings in liblzma-dev, which in turn breaks watchman tests -[debs.not(distro=ubuntu)] -libunwind-dev - -# The current libunwind v1.8.1 release has compiler issues with aarch64 (https://github.com/libunwind/libunwind/issues/702). -# This more recent libunwind version (based on the latest commit, not a release version) got it fixed. -[download] -url = https://github.com/libunwind/libunwind/archive/f081cf42917bdd5c428b77850b473f31f81767cf.tar.gz -sha256 = 4ff5c335c02d225491d6c885db827fb5fa505fee4e68b4d7e866efc0087e7264 - -[build] -builder = autoconf -subdir = libunwind-f081cf42917bdd5c428b77850b473f31f81767cf diff --git a/build/fbcode_builder/manifests/libusb b/build/fbcode_builder/manifests/libusb deleted file mode 100644 index ccbec8053..000000000 --- a/build/fbcode_builder/manifests/libusb +++ /dev/null @@ -1,29 +0,0 @@ -[manifest] -name = libusb - -[debs] -libusb-1.0-0-dev - -[homebrew] -libusb - -[rpms] -libusb-devel -libusb - -[pps] -libusb - -[download] -url = https://github.com/libusb/libusb/releases/download/v1.0.22/libusb-1.0.22.tar.bz2 -sha256 = 75aeb9d59a4fdb800d329a545c2e6799f732362193b465ea198f2aa275518157 - -[build.os=linux] -builder = autoconf -subdir = libusb-1.0.22 - -[autoconf.args] -# fboss (which added the libusb dep) doesn't need udev so it is disabled here. -# if someone in the future wants to add udev for something else, it won't hurt -# fboss. ---disable-udev diff --git a/build/fbcode_builder/manifests/libyaml b/build/fbcode_builder/manifests/libyaml deleted file mode 100644 index b26b519ed..000000000 --- a/build/fbcode_builder/manifests/libyaml +++ /dev/null @@ -1,13 +0,0 @@ -[manifest] -name = libyaml - -[download] -url = https://pyyaml.org/download/libyaml/yaml-0.1.7.tar.gz -sha256 = 8088e457264a98ba451a90b8661fcb4f9d6f478f7265d48322a196cec2480729 - -[build.os=linux] -builder = autoconf -subdir = yaml-0.1.7 - -[build.not(os=linux)] -builder = nop diff --git a/build/fbcode_builder/manifests/llvm b/build/fbcode_builder/manifests/llvm deleted file mode 100644 index 7b069221e..000000000 --- a/build/fbcode_builder/manifests/llvm +++ /dev/null @@ -1,5 +0,0 @@ -[manifest] -name = llvm - -[rpms] -llvm15-devel diff --git a/build/fbcode_builder/manifests/lmdb b/build/fbcode_builder/manifests/lmdb deleted file mode 100644 index 42ca0ab07..000000000 --- a/build/fbcode_builder/manifests/lmdb +++ /dev/null @@ -1,17 +0,0 @@ -[manifest] -name = lmdb - -[build] -builder = make -subdir = lmdb-LMDB_0.9.31/libraries/liblmdb - -[download] -url = https://github.com/LMDB/lmdb/archive/refs/tags/LMDB_0.9.31.tar.gz -sha256 = dd70a8c67807b3b8532b3e987b0a4e998962ecc28643e1af5ec77696b081c9b0 - -[make.build_args] -BUILD_STATIC_ONLY=y - -[make.install_args] -install -BUILD_STATIC_ONLY=y diff --git a/build/fbcode_builder/manifests/lz4 b/build/fbcode_builder/manifests/lz4 deleted file mode 100644 index 0a02f425a..000000000 --- a/build/fbcode_builder/manifests/lz4 +++ /dev/null @@ -1,25 +0,0 @@ -[manifest] -name = lz4 - -[homebrew] -lz4 - -[rpms] -lz4-devel -# centos 8 and centos_stream 9 are missing this rpm -[rpms.not(any(all(distro=centos,distro_vers=8),all(distro=centos_stream,distro_vers=9)))] -lz4-static - -[debs] -liblz4-dev - -[pps] -lz4 - -[download] -url = https://github.com/lz4/lz4/releases/download/v1.10.0/lz4-1.10.0.tar.gz -sha256 = 537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b - -[build] -builder = cmake -subdir = lz4-1.10.0/build/cmake diff --git a/build/fbcode_builder/manifests/lz4-python b/build/fbcode_builder/manifests/lz4-python deleted file mode 100644 index 92cb2ad09..000000000 --- a/build/fbcode_builder/manifests/lz4-python +++ /dev/null @@ -1,31 +0,0 @@ -[manifest] -name = lz4-python - -[homebrew] -lz4 - -[rpms] -lz4-devel -# centos 8 and centos_stream 9 are missing this rpm -[rpms.not(any(all(distro=centos,distro_vers=8),all(distro=centos_stream,distro_vers=9)))] -lz4-static - -[debs] -liblz4-dev - -[pps] -lz4 - -[download] -url = https://github.com/lz4/lz4/releases/download/v1.10.0/lz4-1.10.0.tar.gz -sha256 = 537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b - -[build] -builder = cmake -subdir = lz4-1.10.0/build/cmake - -[build.not(os=linux)] -builder = nop - -[cmake.defines] -CMAKE_POSITION_INDEPENDENT_CODE=ON diff --git a/build/fbcode_builder/manifests/magic_enum b/build/fbcode_builder/manifests/magic_enum deleted file mode 100644 index 23231951f..000000000 --- a/build/fbcode_builder/manifests/magic_enum +++ /dev/null @@ -1,14 +0,0 @@ -[manifest] -name = magic_enum - -[download] -url = https://github.com/Neargye/magic_enum/releases/download/v0.9.7/magic_enum-v0.9.7.tar.gz -sha256 = c047bc7ca0b76752168140e7ae9a4a30d72bf6530c196fdfbf5105a39d40cc46 - -[build] -builder = cmake - -[cmake.defines] -MAGIC_ENUM_OPT_BUILD_EXAMPLES = OFF -MAGIC_ENUM_OPT_BUILD_TESTS = OFF -MAGIC_ENUM_OPT_INSTALL = ON diff --git a/build/fbcode_builder/manifests/mcrouter b/build/fbcode_builder/manifests/mcrouter deleted file mode 100644 index 849e8f75d..000000000 --- a/build/fbcode_builder/manifests/mcrouter +++ /dev/null @@ -1,23 +0,0 @@ -[manifest] -name = mcrouter - -[git] -repo_url = https://github.com/facebook/mcrouter.git - -[dependencies] -folly -wangle -fizz -fbthrift -mvfst -ragel - -[build] -builder = cmake -subdir = . - -[cmake.defines.test=on] -BUILD_TESTS=ON - -[cmake.defines.test=off] -BUILD_TESTS=OFF diff --git a/build/fbcode_builder/manifests/mononoke b/build/fbcode_builder/manifests/mononoke deleted file mode 100644 index de1c4cdf5..000000000 --- a/build/fbcode_builder/manifests/mononoke +++ /dev/null @@ -1,55 +0,0 @@ -[manifest] -name = mononoke -fbsource_path = fbcode/eden -shipit_project = eden -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebook/sapling.git - -[build.not(os=windows)] -builder = cargo - -[build.os=windows] -# building Mononoke on windows is not supported -builder = nop - -[cargo] -build_doc = true -workspace_dir = eden/mononoke - -[github.actions] -rust_version = 1.91 -build_type = MinSizeRel - -[shipit.pathmap] -fbcode/configerator/structs/scm/hg = configerator/structs/scm/hg -fbcode/configerator/structs/scm/hg/public_autocargo = configerator/structs/scm/hg -fbcode/configerator/structs/scm/mononoke/public_autocargo = configerator/structs/scm/mononoke -fbcode/configerator/structs/scm/mononoke = configerator/structs/scm/mononoke -fbcode/eden/oss = . -fbcode/eden = eden -fbcode/eden/fs/public_autocargo = eden/fs -fbcode/eden/mononoke/public_autocargo = eden/mononoke -fbcode/eden/scm/public_autocargo = eden/scm -fbcode/tools/lfs = tools/lfs -tools/rust/ossconfigs = . - -[shipit.strip] -^fbcode/configerator/structs/scm/hg(?!/public_autocargo).*/Cargo\.toml$ -^fbcode/configerator/structs/scm/mononoke(?!/public_autocargo).*/Cargo\.toml$ -^fbcode/eden/fs(?!/public_autocargo).*/Cargo\.toml$ -^fbcode/eden/scm/lib/third-party/rust/.*/Cargo\.toml$ -^fbcode/eden/mononoke(?!/public_autocargo).*/Cargo\.toml$ -# strip other scm code unrelated to mononoke to prevent triggering unnecessary checks -^fbcode/eden(?!/mononoke|/scm/(lib|public_autocargo))/.*$ -^.*/facebook/.*$ -^.*/fb/.*$ - -[dependencies] -fb303 -fbthrift -rust-shed - -[dependencies.fb=on] -rust diff --git a/build/fbcode_builder/manifests/mononoke_integration b/build/fbcode_builder/manifests/mononoke_integration deleted file mode 100644 index 09af47d69..000000000 --- a/build/fbcode_builder/manifests/mononoke_integration +++ /dev/null @@ -1,47 +0,0 @@ -[manifest] -name = mononoke_integration -fbsource_path = fbcode/eden -shipit_project = eden -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebook/sapling.git - -[build.not(os=windows)] -builder = make -subdir = eden/mononoke/tests/integration - -[build.os=windows] -# building Mononoke on windows is not supported -builder = nop - -[make.build_args] -build-getdeps - -[make.install_args] -install-getdeps - -[make.test_args] -test-getdeps - -[shipit.pathmap] -fbcode/eden/mononoke/tests/integration = eden/mononoke/tests/integration - -[shipit.strip] -^.*/facebook/.*$ -^.*/fb/.*$ - -[dependencies] -git-lfs -jq -mononoke -nmap -python -python-click -ripgrep -sapling -tree -zstd - -[dependencies.os=linux] -sqlite3 diff --git a/build/fbcode_builder/manifests/moxygen b/build/fbcode_builder/manifests/moxygen deleted file mode 100644 index 66f9bc6a1..000000000 --- a/build/fbcode_builder/manifests/moxygen +++ /dev/null @@ -1,39 +0,0 @@ -[manifest] -name = moxygen -fbsource_path = fbcode/moxygen -shipit_project = moxygen -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebookexperimental/moxygen.git - -[build.os=windows] -builder = nop - -[build] -builder = cmake -subdir = . -job_weight_mib = 3072 -rewrite_includes = true - -[cmake.defines.test=on] -BUILD_TESTS = ON - -[cmake.defines.test=off] -BUILD_TESTS = OFF - -[dependencies] -zlib -gperf -folly -fizz -wangle -mvfst -proxygen - -[dependencies.test=on] -googletest - -[shipit.pathmap] -fbcode/ti/experimental/moxygen/project_root = . -fbcode/ti/experimental/moxygen = moxygen diff --git a/build/fbcode_builder/manifests/mvfst b/build/fbcode_builder/manifests/mvfst deleted file mode 100644 index c2a797be2..000000000 --- a/build/fbcode_builder/manifests/mvfst +++ /dev/null @@ -1,32 +0,0 @@ -[manifest] -name = mvfst -fbsource_path = fbcode/quic -shipit_project = mvfst -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebook/mvfst.git - -[build] -builder = cmake -subdir = . - -[cmake.defines.test=on] -BUILD_TESTS = ON - -[cmake.defines.all(os=windows, test=on)] -BUILD_TESTS = OFF - -[cmake.defines.test=off] -BUILD_TESTS = OFF - -[dependencies] -folly -fizz - -[dependencies.all(test=on, not(os=windows))] -googletest - -[shipit.pathmap] -fbcode/quic/public_root = . -fbcode/quic = quic diff --git a/build/fbcode_builder/manifests/mvfst-python b/build/fbcode_builder/manifests/mvfst-python deleted file mode 100644 index 641d7fa29..000000000 --- a/build/fbcode_builder/manifests/mvfst-python +++ /dev/null @@ -1,39 +0,0 @@ -[manifest] -name = mvfst-python -fbsource_path = fbcode/quic -shipit_project = mvfst -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebook/mvfst.git - -[build] -builder = cmake -subdir = . - -[build.not(os=linux)] -builder = nop - -[cmake.defines.test=on] -BUILD_TESTS = ON - -[cmake.defines.all(os=windows, test=on)] -BUILD_TESTS = OFF - -[cmake.defines.test=off] -BUILD_TESTS = OFF - -[cmake.defines.os=linux] -CMAKE_POSITION_INDEPENDENT_CODE = ON -BUILD_SHARED_LIBS = ON - -[dependencies] -folly-python -fizz-python - -[dependencies.all(test=on, not(os=windows))] -googletest - -[shipit.pathmap] -fbcode/quic/public_root = . -fbcode/quic = quic diff --git a/build/fbcode_builder/manifests/ncurses b/build/fbcode_builder/manifests/ncurses deleted file mode 100644 index e50c3103b..000000000 --- a/build/fbcode_builder/manifests/ncurses +++ /dev/null @@ -1,30 +0,0 @@ -[manifest] -name = ncurses - -[debs] -libncurses-dev - -[homebrew] -ncurses - -[rpms] -ncurses-devel - -[download] -url = https://ftpmirror.gnu.org/gnu/ncurses/ncurses-6.3.tar.gz -sha256 = 97fc51ac2b085d4cde31ef4d2c3122c21abc217e9090a43a30fc5ec21684e059 - -[build.not(os=windows)] -builder = autoconf -subdir = ncurses-6.3 - -[autoconf.args] ---without-cxx-binding ---without-ada - -[autoconf.args.os=linux] ---enable-shared ---with-shared - -[build.os=windows] -builder = nop diff --git a/build/fbcode_builder/manifests/nghttp2 b/build/fbcode_builder/manifests/nghttp2 deleted file mode 100644 index f2b3f6b31..000000000 --- a/build/fbcode_builder/manifests/nghttp2 +++ /dev/null @@ -1,24 +0,0 @@ -[manifest] -name = nghttp2 - -[rpms] -libnghttp2-devel -libnghttp2 - -[debs] -libnghttp2-dev - -[pps] -libnghttp2 - -[download] -url = https://github.com/nghttp2/nghttp2/releases/download/v1.47.0/nghttp2-1.47.0.tar.gz -sha256 = 62f50f0e9fc479e48b34e1526df8dd2e94136de4c426b7680048181606832b7c - -[build] -builder = autoconf -subdir = nghttp2-1.47.0 - -[autoconf.args] ---enable-lib-only ---disable-dependency-tracking diff --git a/build/fbcode_builder/manifests/ninja b/build/fbcode_builder/manifests/ninja deleted file mode 100644 index 45d837043..000000000 --- a/build/fbcode_builder/manifests/ninja +++ /dev/null @@ -1,32 +0,0 @@ -[manifest] -name = ninja - -[debs] -ninja-build - -[homebrew] -ninja - -[rpms] -ninja-build - -[pps] -ninja - -[download.os=windows] -url = https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-win.zip -sha256 = f550fec705b6d6ff58f2db3c374c2277a37691678d6aba463adcbb129108467a - -[build.os=windows] -builder = nop - -[install.files.os=windows] -ninja.exe = bin/ninja.exe - -[download.not(os=windows)] -url = https://github.com/ninja-build/ninja/archive/v1.12.1.tar.gz -sha256 = 821bdff48a3f683bc4bb3b6f0b5fe7b2d647cf65d52aeb63328c91a6c6df285a - -[build.not(os=windows)] -builder = ninja_bootstrap -subdir = ninja-1.12.1 \ No newline at end of file diff --git a/build/fbcode_builder/manifests/nlohmann-json b/build/fbcode_builder/manifests/nlohmann-json deleted file mode 100644 index 7d552d95f..000000000 --- a/build/fbcode_builder/manifests/nlohmann-json +++ /dev/null @@ -1,12 +0,0 @@ -[manifest] -name = nlohmann-json - -[download] -url = https://github.com/nlohmann/json/archive/refs/tags/v3.10.5.tar.gz -sha256 = 5daca6ca216495edf89d167f808d1d03c4a4d929cef7da5e10f135ae1540c7e4 - -[dependencies] - -[build] -builder = cmake -subdir = json-3.10.5 diff --git a/build/fbcode_builder/manifests/nmap b/build/fbcode_builder/manifests/nmap deleted file mode 100644 index 3e935177b..000000000 --- a/build/fbcode_builder/manifests/nmap +++ /dev/null @@ -1,30 +0,0 @@ -[manifest] -name = nmap - -[rpms] -nmap -nmap-ncat - -[debs] -nmap - -# 18.04 combines ncat into the nmap package, newer need the separate one -[debs.not(all(distro=ubuntu,distro_vers="18.04"))] -ncat - -[download.not(os=windows)] -url = https://api.github.com/repos/nmap/nmap/tarball/ef8213a36c2e89233c806753a57b5cd473605408 -sha256 = eda39e5a8ef4964fac7db16abf91cc11ff568eac0fa2d680b0bfa33b0ed71f4a - -[build.not(os=windows)] -builder = autoconf -subdir = nmap-nmap-ef8213a -build_in_src_dir = true - -[build.os=windows] -builder = nop - -[autoconf.args] -# Without this option the build was filing to find some third party libraries -# that we don't need -enable_rdma=no diff --git a/build/fbcode_builder/manifests/numa b/build/fbcode_builder/manifests/numa deleted file mode 100644 index d57b8afab..000000000 --- a/build/fbcode_builder/manifests/numa +++ /dev/null @@ -1,13 +0,0 @@ -[manifest] -name = numa - -[download] -url = https://github.com/numactl/numactl/releases/download/v2.0.19/numactl-2.0.19.tar.gz -sha256 = f2672a0381cb59196e9c246bf8bcc43d5568bc457700a697f1a1df762b9af884 - -[build] -builder = autoconf -subdir = numactl-2.0.19 - -[rpms.distro=centos_stream] -numactl-devel diff --git a/build/fbcode_builder/manifests/openr b/build/fbcode_builder/manifests/openr deleted file mode 100644 index 913d81f37..000000000 --- a/build/fbcode_builder/manifests/openr +++ /dev/null @@ -1,38 +0,0 @@ -[manifest] -name = openr -fbsource_path = facebook/openr -shipit_project = openr -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebook/openr.git - -[build.os=linux] -builder = cmake -# openr files take a lot of RAM to compile. -job_weight_mib = 3072 - -[build.not(os=linux)] -# boost.fiber is required and that is not available on macos. -builder = nop - -[dependencies] -boost -fb303 -fbthrift -folly -googletest -re2 -range-v3 - -[cmake.defines.test=on] -BUILD_TESTS=ON -ADD_ROOT_TESTS=OFF - -[cmake.defines.test=off] -BUILD_TESTS=OFF - - -[shipit.pathmap] -fbcode/openr = openr -fbcode/openr/public_tld = . diff --git a/build/fbcode_builder/manifests/openssl b/build/fbcode_builder/manifests/openssl deleted file mode 100644 index ebd680e7e..000000000 --- a/build/fbcode_builder/manifests/openssl +++ /dev/null @@ -1,35 +0,0 @@ -[manifest] -name = openssl - -[debs] -libssl-dev - -[homebrew] -openssl -# on homebrew need the matching curl and ca- - -[rpms] -openssl -openssl-devel -openssl-libs - -[pps] -openssl - -# no need to download on the systems where we always use the system libs -[download.not(any(os=linux, os=freebsd))] -# match the openssl version packages in ubuntu LTS folly current supports -url = https://www.openssl.org/source/openssl-3.0.15.tar.gz -sha256 = 23c666d0edf20f14249b3d8f0368acaee9ab585b09e1de82107c66e1f3ec9533 - -# We use the system openssl on these platforms even without --allow-system-packages -[build.any(os=linux, os=freebsd)] -builder = nop - -[build.not(any(os=linux, os=freebsd))] -builder = openssl -subdir = openssl-3.0.15 - -[dependencies.os=windows] -jom -perl diff --git a/build/fbcode_builder/manifests/osxfuse b/build/fbcode_builder/manifests/osxfuse deleted file mode 100644 index b6c6c551f..000000000 --- a/build/fbcode_builder/manifests/osxfuse +++ /dev/null @@ -1,12 +0,0 @@ -[manifest] -name = osxfuse - -[download] -url = https://github.com/osxfuse/osxfuse/archive/osxfuse-3.8.3.tar.gz -sha256 = 93bab6731bdfe8dc1ef069483437270ce7fe5a370f933d40d8d0ef09ba846c0c - -[build] -builder = nop - -[install.files] -osxfuse-osxfuse-3.8.3/common = include diff --git a/build/fbcode_builder/manifests/patchelf b/build/fbcode_builder/manifests/patchelf deleted file mode 100644 index 7025dc66a..000000000 --- a/build/fbcode_builder/manifests/patchelf +++ /dev/null @@ -1,20 +0,0 @@ -[manifest] -name = patchelf - -[rpms] -patchelf - -[debs] -patchelf - -[pps] -patchelf - -[download] -url = https://github.com/NixOS/patchelf/archive/0.10.tar.gz -sha256 = b3cb6bdedcef5607ce34a350cf0b182eb979f8f7bc31eae55a93a70a3f020d13 - -[build] -builder = autoconf -subdir = patchelf-0.10 - diff --git a/build/fbcode_builder/manifests/pcre2 b/build/fbcode_builder/manifests/pcre2 deleted file mode 100644 index 9ba119a78..000000000 --- a/build/fbcode_builder/manifests/pcre2 +++ /dev/null @@ -1,20 +0,0 @@ -[manifest] -name = pcre2 - -[homebrew] -pcre2 - -[rpms] -pcre2-devel -pcre-static - -[debs] -libpcre2-dev - -[download] -url = https://github.com/PCRE2Project/pcre2/releases/download/pcre2-10.40/pcre2-10.40.tar.bz2 -sha256 = 14e4b83c4783933dc17e964318e6324f7cae1bc75d8f3c79bc6969f00c159d68 - -[build] -builder = cmake -subdir = pcre2-10.40 diff --git a/build/fbcode_builder/manifests/perl b/build/fbcode_builder/manifests/perl deleted file mode 100644 index 9c8bfa31a..000000000 --- a/build/fbcode_builder/manifests/perl +++ /dev/null @@ -1,10 +0,0 @@ -[manifest] -name = perl - -[download.os=windows] -url = https://strawberryperl.com/download/5.28.1.1/strawberry-perl-5.28.1.1-64bit-portable.zip -sha256 = 935c95ba096fa11c4e1b5188732e3832d330a2a79e9882ab7ba8460ddbca810d - -[build.os=windows] -builder = nop -subdir = perl diff --git a/build/fbcode_builder/manifests/pexpect b/build/fbcode_builder/manifests/pexpect deleted file mode 100644 index 682e66a54..000000000 --- a/build/fbcode_builder/manifests/pexpect +++ /dev/null @@ -1,12 +0,0 @@ -[manifest] -name = pexpect - -[download] -url = https://files.pythonhosted.org/packages/0e/3e/377007e3f36ec42f1b84ec322ee12141a9e10d808312e5738f52f80a232c/pexpect-4.7.0-py2.py3-none-any.whl -sha256 = 2094eefdfcf37a1fdbfb9aa090862c1a4878e5c7e0e7e7088bdb511c558e5cd1 - -[build] -builder = python-wheel - -[dependencies] -python-ptyprocess diff --git a/build/fbcode_builder/manifests/proxygen b/build/fbcode_builder/manifests/proxygen deleted file mode 100644 index d92876599..000000000 --- a/build/fbcode_builder/manifests/proxygen +++ /dev/null @@ -1,38 +0,0 @@ -[manifest] -name = proxygen -fbsource_path = fbcode/proxygen -shipit_project = proxygen -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebook/proxygen.git - -[build.os=windows] -builder = nop - -[build] -builder = cmake -subdir = . -job_weight_mib = 3072 - -[cmake.defines.test=on] -BUILD_TESTS = ON - -[cmake.defines.test=off] -BUILD_TESTS = OFF - -[dependencies] -zlib -gperf -folly -fizz -wangle -mvfst -c-ares - -[dependencies.test=on] -googletest - -[shipit.pathmap] -fbcode/proxygen/public_tld = . -fbcode/proxygen = proxygen diff --git a/build/fbcode_builder/manifests/proxygen-python b/build/fbcode_builder/manifests/proxygen-python deleted file mode 100644 index 69cc6a8e4..000000000 --- a/build/fbcode_builder/manifests/proxygen-python +++ /dev/null @@ -1,45 +0,0 @@ -[manifest] -name = proxygen-python -fbsource_path = fbcode/proxygen -shipit_project = proxygen -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebook/proxygen.git - -[build] -builder = cmake -subdir = . -job_weight_mib = 3072 - -[build.not(os=linux)] -builder = nop - -[cmake.defines.test=on] -BUILD_TESTS = ON - -[cmake.defines.test=off] -BUILD_TESTS = OFF - -[cmake.defines] -BUILD_SAMPLES = OFF - -[cmake.defines.os=linux] -CMAKE_POSITION_INDEPENDENT_CODE = ON -BUILD_SHARED_LIBS = ON - -[dependencies] -zlib-python -gperf -folly-python -fizz-python -wangle-python -mvfst-python -c-ares - -[dependencies.test=on] -googletest - -[shipit.pathmap] -fbcode/proxygen/public_tld = . -fbcode/proxygen = proxygen diff --git a/build/fbcode_builder/manifests/python b/build/fbcode_builder/manifests/python deleted file mode 100644 index 017380d57..000000000 --- a/build/fbcode_builder/manifests/python +++ /dev/null @@ -1,51 +0,0 @@ -[manifest] -name = python - -[homebrew] -python@3.10 - -# sapling needs match statements with arrive in python 3.12 in centos 10 -[rpms.not(all(distro=centos_stream,distro_vers=9))] -python3 -python3-devel - -# Centos Stream 9 default python is 3.9, sapling needs 3.10+ -[rpms.all(distro=centos_stream,distro_vers=9)] -python3.12 -python3.12-devel - -# sapling needs match statements with arrive in python 3.10 in ubuntu 22.04 -[debs.not(all(distro=ubuntu,any(distro_vers="18.04",distro_vers="20.04")))] -python3-all-dev - -[pps] -python3 - -[download] -url = https://www.python.org/ftp/python/3.10.19/Python-3.10.19.tgz -sha256 = a078fb2d7a216071ebbe2e34b5f5355dd6b6e9b0cd1bacc4a41c63990c5a0eec - -[build] -builder = autoconf -subdir = Python-3.10.19 - -[autoconf.args] ---enable-shared ---with-ensurepip=install - -# python's pkg-config libffi detection is broken -# See https://bugs.python.org/issue34823 for clearest description -# and pending PR https://github.com/python/cpython/pull/20451 -# The documented workaround requires an environment variable derived from -# pkg-config to be passed into its configure step -[autoconf.envcmd.LDFLAGS] -pkg-config ---libs-only-L -libffi - -[dependencies] -libffi -# eden tests expect the python bz2 support -bz2 -# eden tests expect the python curses support -ncurses diff --git a/build/fbcode_builder/manifests/python-3_14 b/build/fbcode_builder/manifests/python-3_14 deleted file mode 100644 index d2169a1e3..000000000 --- a/build/fbcode_builder/manifests/python-3_14 +++ /dev/null @@ -1,12 +0,0 @@ -# This is primarily to support CinderX's CI, so it's not heavily configured. - -[manifest] -name = python-3_14 - -[download] -url = https://github.com/python/cpython/archive/refs/tags/v3.14.3.tar.gz -sha256 = f229a232052ae318d2fc8eb0aca4a02d631e7e1a8790ef1f9b65e1632743a469 - -[build] -builder = autoconf -subdir = cpython-3.14.3 diff --git a/build/fbcode_builder/manifests/python-click b/build/fbcode_builder/manifests/python-click deleted file mode 100644 index cdf29c4d0..000000000 --- a/build/fbcode_builder/manifests/python-click +++ /dev/null @@ -1,15 +0,0 @@ -[manifest] -name = python-click - -[download] -url = https://files.pythonhosted.org/packages/d2/3d/fa76db83bf75c4f8d338c2fd15c8d33fdd7ad23a9b5e57eb6c5de26b430e/click-7.1.2-py2.py3-none-any.whl -sha256 = dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc - -[build] -builder = python-wheel - -[rpms] -python3-click - -[debs] -python3-click diff --git a/build/fbcode_builder/manifests/python-filelock b/build/fbcode_builder/manifests/python-filelock deleted file mode 100644 index 40502de7c..000000000 --- a/build/fbcode_builder/manifests/python-filelock +++ /dev/null @@ -1,9 +0,0 @@ -[manifest] -name = python-filelock - -[download] -url = https://files.pythonhosted.org/packages/31/24/ee722b92f23b9ebd87783e893a75352c048bbbc1f67dce0d63b58b46cb48/filelock-3.3.2-py3-none-any.whl -sha256 = bb2a1c717df74c48a2d00ed625e5a66f8572a3a30baacb7657add1d7bac4097b - -[build] -builder = python-wheel diff --git a/build/fbcode_builder/manifests/python-main b/build/fbcode_builder/manifests/python-main deleted file mode 100644 index d3dff19d9..000000000 --- a/build/fbcode_builder/manifests/python-main +++ /dev/null @@ -1,18 +0,0 @@ -# This is primarily to support CinderX's CI, so it's not heavily configured. - -[manifest] -name = python-main -fbsource_path = third-party/python/main/pristine -# We don't actually have a shipit project for python-main, but we use getdeps -# built-in shipit implementation which just needs a shipit.pathmap. -shipit_project = dummy-name - - -[git] -repo_url = https://github.com/python/cpython.git - -[shipit.pathmap] -third-party/python/main/pristine = . - -[build] -builder = autoconf diff --git a/build/fbcode_builder/manifests/python-psutil b/build/fbcode_builder/manifests/python-psutil deleted file mode 100644 index 921781fc0..000000000 --- a/build/fbcode_builder/manifests/python-psutil +++ /dev/null @@ -1,10 +0,0 @@ -[manifest] -name = python-psutil - -[download] -url = https://files.pythonhosted.org/packages/bf/b9/b0eb3f3cbcb734d930fdf839431606844a825b23eaf9a6ab371edac8162c/psutil-7.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl -sha256 = 4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34 - -[build] -builder = python-wheel - diff --git a/build/fbcode_builder/manifests/python-ptyprocess b/build/fbcode_builder/manifests/python-ptyprocess deleted file mode 100644 index adc60e048..000000000 --- a/build/fbcode_builder/manifests/python-ptyprocess +++ /dev/null @@ -1,9 +0,0 @@ -[manifest] -name = python-ptyprocess - -[download] -url = https://files.pythonhosted.org/packages/d1/29/605c2cc68a9992d18dada28206eeada56ea4bd07a239669da41674648b6f/ptyprocess-0.6.0-py2.py3-none-any.whl -sha256 = d7cc528d76e76342423ca640335bd3633420dc1366f258cb31d05e865ef5ca1f - -[build] -builder = python-wheel diff --git a/build/fbcode_builder/manifests/python-pyyaml b/build/fbcode_builder/manifests/python-pyyaml deleted file mode 100644 index 5c40a16e6..000000000 --- a/build/fbcode_builder/manifests/python-pyyaml +++ /dev/null @@ -1,9 +0,0 @@ -[manifest] -name = python-pyyaml - -[download] -url = https://files.pythonhosted.org/packages/25/a2/b725b61ac76a75583ae7104b3209f75ea44b13cfd026aa535ece22b7f22e/PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl -sha256 = 22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 - -[build] -builder = python-wheel diff --git a/build/fbcode_builder/manifests/python-setuptools b/build/fbcode_builder/manifests/python-setuptools deleted file mode 100644 index 1f6013ddc..000000000 --- a/build/fbcode_builder/manifests/python-setuptools +++ /dev/null @@ -1,22 +0,0 @@ -[manifest] -name = python-setuptools - -[download] -url = https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl -sha256 = 062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922 - -[build] -builder = python-wheel - -[rpms] -python3-setuptools - -# Centos Stream 9 default python is 3.9, sapling needs 3.10+ -[rpms.all(distro=centos_stream,distro_vers=9)] -python3.12-setuptools - -[homebrew] -python-setuptools - -[debs] -python3-setuptools diff --git a/build/fbcode_builder/manifests/python-setuptools-69 b/build/fbcode_builder/manifests/python-setuptools-69 deleted file mode 100644 index 43834f9b6..000000000 --- a/build/fbcode_builder/manifests/python-setuptools-69 +++ /dev/null @@ -1,18 +0,0 @@ -[manifest] -name = python-setuptools-69 - -[download] -url = https://files.pythonhosted.org/packages/c0/7a/3da654f49c95d0cc6e9549a855b5818e66a917e852ec608e77550c8dc08b/setuptools-69.1.1-py3-none-any.whl -sha256 = 02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56 - -[build] -builder = python-wheel - -[rpms] -python3-setuptools - -[homebrew] -python-setuptools - -[debs] -python3-setuptools diff --git a/build/fbcode_builder/manifests/python-six b/build/fbcode_builder/manifests/python-six deleted file mode 100644 index a712188dc..000000000 --- a/build/fbcode_builder/manifests/python-six +++ /dev/null @@ -1,9 +0,0 @@ -[manifest] -name = python-six - -[download] -url = https://files.pythonhosted.org/packages/73/fb/00a976f728d0d1fecfe898238ce23f502a721c0ac0ecfedb80e0d88c64e9/six-1.12.0-py2.py3-none-any.whl -sha256 = 3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c - -[build] -builder = python-wheel diff --git a/build/fbcode_builder/manifests/python-toml b/build/fbcode_builder/manifests/python-toml deleted file mode 100644 index b49a3b8fb..000000000 --- a/build/fbcode_builder/manifests/python-toml +++ /dev/null @@ -1,9 +0,0 @@ -[manifest] -name = python-toml - -[download] -url = https://files.pythonhosted.org/packages/a2/12/ced7105d2de62fa7c8fb5fce92cc4ce66b57c95fb875e9318dba7f8c5db0/toml-0.10.0-py2.py3-none-any.whl -sha256 = 235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e - -[build] -builder = python-wheel diff --git a/build/fbcode_builder/manifests/ragel b/build/fbcode_builder/manifests/ragel deleted file mode 100644 index 336a39b20..000000000 --- a/build/fbcode_builder/manifests/ragel +++ /dev/null @@ -1,19 +0,0 @@ -[manifest] -name = ragel - -[debs] -ragel - -[homebrew] -ragel - -[rpms] -ragel - -[download] -url = https://www.colm.net/files/ragel/ragel-6.10.tar.gz -sha256 = 5f156edb65d20b856d638dd9ee2dfb43285914d9aa2b6ec779dac0270cd56c3f - -[build] -builder = autoconf -subdir = ragel-6.10 diff --git a/build/fbcode_builder/manifests/range-v3 b/build/fbcode_builder/manifests/range-v3 deleted file mode 100644 index e3778a368..000000000 --- a/build/fbcode_builder/manifests/range-v3 +++ /dev/null @@ -1,14 +0,0 @@ -[manifest] -name = range-v3 - -[download] -url = https://github.com/ericniebler/range-v3/archive/refs/tags/0.11.0.tar.gz -sha256 = 376376615dbba43d3bef75aa590931431ecb49eb36d07bb726a19f680c75e20c - - -[build] -builder = cmake -subdir = range-v3-0.11.0 - -[cmake.defines] -RANGE_V3_EXAMPLES=OFF diff --git a/build/fbcode_builder/manifests/rdma-core b/build/fbcode_builder/manifests/rdma-core deleted file mode 100644 index cf7b2200d..000000000 --- a/build/fbcode_builder/manifests/rdma-core +++ /dev/null @@ -1,8 +0,0 @@ -[manifest] -name = rdma-core - -[debs] -rdma-core - -[rpms] -rdma-core-devel diff --git a/build/fbcode_builder/manifests/re2 b/build/fbcode_builder/manifests/re2 deleted file mode 100644 index 1fe1eccbd..000000000 --- a/build/fbcode_builder/manifests/re2 +++ /dev/null @@ -1,23 +0,0 @@ -[manifest] -name = re2 - -[homebrew] -re2 - -[debs] -libre2-dev - -[rpms] -re2 -re2-devel - -[pps] -re2 - -[download] -url = https://github.com/google/re2/archive/2020-11-01.tar.gz -sha256 = 8903cc66c9d34c72e2bc91722288ebc7e3ec37787ecfef44d204b2d6281954d7 - -[build] -builder = cmake -subdir = re2-2020-11-01 diff --git a/build/fbcode_builder/manifests/rebalancer b/build/fbcode_builder/manifests/rebalancer deleted file mode 100644 index bde55cffe..000000000 --- a/build/fbcode_builder/manifests/rebalancer +++ /dev/null @@ -1,30 +0,0 @@ -[manifest] -name = rebalancer -fbsource_path = fbcode/algopt/rebalancer/ - -[git] -# To git clone on devserver, setup fwdproxy: -# https://www.internalfb.com/wiki/Open_Source/Maintain_a_FB_OSS_Project/Devserver_GitHub_Access/ -repo_url = git@github.com:facebookincubator/rebalancer.git -branch = richard - -[build] -builder = cmake - -[dependencies] -boost -folly -gflags -glog -fbthrift -fmt -fizz -googletest -xxhash - -[cmake.defines.os=darwin] -REBALANCER_USE_SCIP=0 - -[shipit.pathmap] -fbcode/algopt/rebalancer/oss = . -fbcode/algopt/rebalancer = rebalancer diff --git a/build/fbcode_builder/manifests/ripgrep b/build/fbcode_builder/manifests/ripgrep deleted file mode 100644 index 140a4e8af..000000000 --- a/build/fbcode_builder/manifests/ripgrep +++ /dev/null @@ -1,15 +0,0 @@ -[manifest] -name = ripgrep - -[rpms] -ripgrep - -[debs] -ripgrep - -[homebrew] -ripgrep - -# only used from system packages currently -[build] -builder = nop diff --git a/build/fbcode_builder/manifests/rocksdb b/build/fbcode_builder/manifests/rocksdb deleted file mode 100644 index c56066378..000000000 --- a/build/fbcode_builder/manifests/rocksdb +++ /dev/null @@ -1,38 +0,0 @@ -[manifest] -name = rocksdb - -[download] -url = https://github.com/facebook/rocksdb/archive/refs/tags/v8.7.3.zip -sha256 = 36c06b61dc167f2455990d60dd88d734b73aa8c4dfc095243efd0243834c6cd3 - -[dependencies] -lz4 -snappy - -[build] -builder = cmake -subdir = rocksdb-8.7.3 - -[cmake.defines] -WITH_SNAPPY=ON -WITH_LZ4=ON -WITH_TESTS=OFF -WITH_BENCHMARK_TOOLS=OFF -# We get relocation errors with the static gflags lib, -# and there's no clear way to make it pick the shared gflags -# so just turn it off. -WITH_GFLAGS=OFF -# Disable the use of -Werror -FAIL_ON_WARNINGS = OFF - -[cmake.defines.os=windows] -ROCKSDB_INSTALL_ON_WINDOWS=ON -# RocksDB hard codes the paths to the snappy libs to something -# that doesn't exist; ignoring the usual cmake rules. As a result, -# we can't build it with snappy without either patching rocksdb or -# without introducing more complex logic to the build system to -# connect the snappy build outputs to rocksdb's custom logic here. -# Let's just turn it off on windows. -WITH_SNAPPY=OFF -WITH_LZ4=ON -ROCKSDB_SKIP_THIRDPARTY=ON diff --git a/build/fbcode_builder/manifests/rust-shed b/build/fbcode_builder/manifests/rust-shed deleted file mode 100644 index 31e2b61d9..000000000 --- a/build/fbcode_builder/manifests/rust-shed +++ /dev/null @@ -1,35 +0,0 @@ -[manifest] -name = rust-shed -fbsource_path = fbcode/common/rust/shed -shipit_project = rust-shed -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebookexperimental/rust-shed.git - -[build] -builder = cargo - -[cargo] -build_doc = true -workspace_dir = - -[shipit.pathmap] -fbcode/common/rust/shed = shed -fbcode/common/rust/shed/public_autocargo = shed -fbcode/common/rust/shed/public_tld = . -tools/rust/ossconfigs = . - -[shipit.strip] -^fbcode/common/rust/shed/(?!public_autocargo|public_tld).+/Cargo\.toml$ - -[dependencies] -fbthrift -fb303 - -# We use the system openssl on linux -[dependencies.not(os=linux)] -openssl - -[dependencies.fbsource=on] -rust diff --git a/build/fbcode_builder/manifests/sapling b/build/fbcode_builder/manifests/sapling deleted file mode 100644 index cff882c67..000000000 --- a/build/fbcode_builder/manifests/sapling +++ /dev/null @@ -1,75 +0,0 @@ -[manifest] -name = sapling -fbsource_path = fbcode/eden -shipit_project = eden -shipit_fbcode_builder = true - -[github.actions] -required_locales = en_US.UTF-8 - -[git] -repo_url = https://github.com/facebook/sapling.git - -[build.not(os=windows)] -builder = make -subdir = eden/scm - -[build.os=windows] -# For now the biggest blocker is missing "make" on windows, but there are bound -# to be more -builder = nop - -[make.build_args] -getdepsbuild - -[make.install_args] -install-getdeps - -[make.test_args] -test-getdeps - -[shipit.pathmap] -fbcode/configerator/structs/scm/hg = configerator/structs/scm/hg -fbcode/configerator/structs/scm/hg/public_autocargo = configerator/structs/scm/hg -fbcode/eden/oss = . -fbcode/eden = eden -fbcode/eden/fs/public_autocargo = eden/fs -fbcode/eden/mononoke/public_autocargo = eden/mononoke -fbcode/eden/scm/public_autocargo = eden/scm -fbcode/tools/lfs = tools/lfs - -[shipit.strip] -^fbcode/configerator/structs/scm/hg(?!/public_autocargo).*/Cargo\.toml$ -^fbcode/eden/addons/.*$ -^fbcode/eden/fs/eden-config\.h$ -^fbcode/eden/fs/py/eden/config\.py$ -^fbcode/eden/hg-server/.*$ -^fbcode/eden/fs(?!/public_autocargo).*/Cargo\.toml$ -^fbcode/eden/mononoke(?!/public_autocargo).*/Cargo\.toml$ -^fbcode/eden/scm(?!/public_autocargo|/edenscmnative/bindings).*/Cargo\.toml$ -^fbcode/eden/scm/build/.*$ -^fbcode/eden/website/.*$ -^fbcode/eden/.*/\.cargo/.*$ -^.*/facebook/.*$ -^.*/fb/.*$ -/Cargo\.lock$ -\.pyc$ - -[dependencies] -fb303 -fbthrift -rust-shed - -[dependencies.all(test=on,not(os=darwin))] -hexdump - -[dependencies.not(os=windows)] -python -python-setuptools - -# We use the system openssl on linux -[dependencies.not(os=linux)] -openssl - -[dependencies.fbsource=on] -rust diff --git a/build/fbcode_builder/manifests/snappy b/build/fbcode_builder/manifests/snappy deleted file mode 100644 index c458a0ae8..000000000 --- a/build/fbcode_builder/manifests/snappy +++ /dev/null @@ -1,30 +0,0 @@ -[manifest] -name = snappy - -[homebrew] -snappy - -[debs] -libsnappy-dev - -[rpms] -snappy-devel - -[pps] -snappy - -[download] -url = https://github.com/google/snappy/archive/1.1.7.tar.gz -sha256 = 3dfa02e873ff51a11ee02b9ca391807f0c8ea0529a4924afa645fbf97163f9d4 - -[build] -builder = cmake -subdir = snappy-1.1.7 - -[cmake.defines] -SNAPPY_BUILD_TESTS = OFF - -# Avoid problems like `relocation R_X86_64_PC32 against symbol` on ELF systems -# when linking rocksdb, which builds PIC even when building a static lib -[cmake.defines.os=linux] -BUILD_SHARED_LIBS = ON diff --git a/build/fbcode_builder/manifests/sparsemap b/build/fbcode_builder/manifests/sparsemap deleted file mode 100644 index 330b6439f..000000000 --- a/build/fbcode_builder/manifests/sparsemap +++ /dev/null @@ -1,10 +0,0 @@ -[manifest] -name = sparsemap - -[download] -url = https://github.com/Tessil/sparse-map/archive/refs/tags/v0.6.2.tar.gz -sha256 = 7020c21e8752e59d72e37456cd80000e18671c803890a3e55ae36b295eba99f6 - -[build] -builder = cmake -subdir = sparse-map-0.6.2/ diff --git a/build/fbcode_builder/manifests/sqlite3 b/build/fbcode_builder/manifests/sqlite3 deleted file mode 100644 index 5a983f8aa..000000000 --- a/build/fbcode_builder/manifests/sqlite3 +++ /dev/null @@ -1,29 +0,0 @@ -[manifest] -name = sqlite3 - -[debs] -libsqlite3-dev -sqlite3 - -[homebrew] -sqlite - -[rpms] -sqlite-devel -sqlite-libs -sqlite - -[pps] -sqlite3 - -[download] -url = https://sqlite.org/2019/sqlite-amalgamation-3280000.zip -sha256 = d02fc4e95cfef672b45052e221617a050b7f2e20103661cda88387349a9b1327 - -[dependencies] -cmake -ninja - -[build] -builder = sqlite -subdir = sqlite-amalgamation-3280000 diff --git a/build/fbcode_builder/manifests/systemd b/build/fbcode_builder/manifests/systemd deleted file mode 100644 index d39d13f0c..000000000 --- a/build/fbcode_builder/manifests/systemd +++ /dev/null @@ -1,19 +0,0 @@ -[manifest] -name = systemd - -[rpms] -systemd -systemd-devel - - -[download] -url = https://github.com/systemd/systemd/archive/refs/tags/v256.7.tar.gz -sha256 = 896d76ff65c88f5fd9e42f90d152b0579049158a163431dd77cdc57748b1d7b0 - -[build.os=linux] -builder = meson -subdir = systemd-256.7 - -[meson.setup_args] --Dstatic-libsystemd=true --Dprefix=/ diff --git a/build/fbcode_builder/manifests/tabulate b/build/fbcode_builder/manifests/tabulate deleted file mode 100644 index 8781f37e8..000000000 --- a/build/fbcode_builder/manifests/tabulate +++ /dev/null @@ -1,14 +0,0 @@ -[manifest] -name = tabulate - -[download] -url = https://github.com/p-ranav/tabulate/archive/refs/tags/v1.5.tar.gz -sha256 = 16b289f46306283544bb593f4601e80d6ea51248fde52e910cc569ef08eba3fb - -[build] -builder = cmake -subdir = tabulate-1.5 - -[cmake.defines] -tabulate_BUILD_TESTS = OFF -tabulate_BUILD_SAMPLES = OFF diff --git a/build/fbcode_builder/manifests/tree b/build/fbcode_builder/manifests/tree deleted file mode 100644 index ccd0180a7..000000000 --- a/build/fbcode_builder/manifests/tree +++ /dev/null @@ -1,37 +0,0 @@ -[manifest] -name = tree - -[debs] -tree - -[homebrew] -tree - -[rpms] -tree - -[download.os=linux] -url = https://salsa.debian.org/debian/tree-packaging/-/archive/debian/1.8.0-1/tree-packaging-debian-1.8.0-1.tar.gz -sha256 = a841eee1d52bfd64a48f54caab9937b9bd92935055c48885c4ab1ae4dab7fae5 - -[download.os=darwin] -# The official package of tree source requires users of non-Linux platform to -# comment/uncomment certain lines in the Makefile to build for their platform. -# Besauce getdeps.py doesn't have that functionality we just use this custom -# fork of tree which has proper lines uncommented for a OSX build -url = https://github.com/lukaspiatkowski/tree-command/archive/debian/1.8.0-1-macos.tar.gz -sha256 = 9cbe889553d95cf5a2791dd0743795d46a3c092c5bba691769c0e5c52e11229e - -[build.os=linux] -builder = make -subdir = tree-packaging-debian-1.8.0-1 - -[build.os=darwin] -builder = make -subdir = tree-command-debian-1.8.0-1-macos - -[build.os=windows] -builder = nop - -[make.install_args] -install diff --git a/build/fbcode_builder/manifests/wangle b/build/fbcode_builder/manifests/wangle deleted file mode 100644 index 6b330d620..000000000 --- a/build/fbcode_builder/manifests/wangle +++ /dev/null @@ -1,27 +0,0 @@ -[manifest] -name = wangle -fbsource_path = fbcode/wangle -shipit_project = wangle -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebook/wangle.git - -[build] -builder = cmake -subdir = wangle - -[cmake.defines.test=on] -BUILD_TESTS=ON - -[cmake.defines.test=off] -BUILD_TESTS=OFF - -[dependencies] -folly -googletest -fizz - -[shipit.pathmap] -fbcode/wangle/public_tld = . -fbcode/wangle = wangle diff --git a/build/fbcode_builder/manifests/wangle-python b/build/fbcode_builder/manifests/wangle-python deleted file mode 100644 index b3679962e..000000000 --- a/build/fbcode_builder/manifests/wangle-python +++ /dev/null @@ -1,34 +0,0 @@ -[manifest] -name = wangle-python -fbsource_path = fbcode/wangle -shipit_project = wangle -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebook/wangle.git - -[build] -builder = cmake -subdir = wangle - -[build.not(os=linux)] -builder = nop - -[cmake.defines.test=on] -BUILD_TESTS=ON - -[cmake.defines.test=off] -BUILD_TESTS=OFF - -[cmake.defines.os=linux] -CMAKE_POSITION_INDEPENDENT_CODE = ON -BUILD_SHARED_LIBS = ON - -[dependencies] -folly-python -googletest -fizz-python - -[shipit.pathmap] -fbcode/wangle/public_tld = . -fbcode/wangle = wangle diff --git a/build/fbcode_builder/manifests/watchman b/build/fbcode_builder/manifests/watchman deleted file mode 100644 index 6c42e911c..000000000 --- a/build/fbcode_builder/manifests/watchman +++ /dev/null @@ -1,48 +0,0 @@ -[manifest] -name = watchman -fbsource_path = fbcode/watchman -shipit_project = watchman -shipit_fbcode_builder = true - -[git] -repo_url = https://github.com/facebook/watchman.git - -[build] -builder = cmake - -[dependencies] -boost -cpptoml -edencommon -fb303 -fbthrift -folly -pcre2 -googletest -python-setuptools-69 - -[dependencies.fbsource=on] -rust - -[shipit.pathmap] -fbcode/watchman = watchman -fbcode/watchman/oss = . -fbcode/eden/fs = eden/fs - -[shipit.strip] -^fbcode/eden/fs/(?!.*\.thrift|service/shipit_test_file\.txt) - -[cmake.defines.fb=on] -ENABLE_EDEN_SUPPORT=ON -IS_FB_BUILD=ON - -# FB macos specific settings -[cmake.defines.all(fb=on,os=darwin)] -# this path is coupled with the FB internal watchman-osx.spec -WATCHMAN_STATE_DIR=/opt/facebook/watchman/var/run/watchman -# tell cmake not to try to create /opt/facebook/... -INSTALL_WATCHMAN_STATE_DIR=OFF -USE_SYS_PYTHON=OFF - -[depends.environment] -WATCHMAN_VERSION_OVERRIDE diff --git a/build/fbcode_builder/manifests/xxhash b/build/fbcode_builder/manifests/xxhash deleted file mode 100644 index b06002f66..000000000 --- a/build/fbcode_builder/manifests/xxhash +++ /dev/null @@ -1,30 +0,0 @@ -[manifest] -name = xxhash - -[download] -url = https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.2.tar.gz -sha256 = baee0c6afd4f03165de7a4e67988d16f0f2b257b51d0e3cb91909302a26a79c4 - -[rpms] -xxhash-devel - -[debs] -libxxhash-dev -xxhash - -[homebrew] -xxhash - -[build.not(os=windows)] -builder = make -subdir = xxHash-0.8.2 - -[make.build_args] -all - -[make.install_args] -install - -[build.os=windows] -builder = cmake -subdir = xxHash-0.8.2/cmake_unofficial diff --git a/build/fbcode_builder/manifests/xz b/build/fbcode_builder/manifests/xz deleted file mode 100644 index 6552f2871..000000000 --- a/build/fbcode_builder/manifests/xz +++ /dev/null @@ -1,20 +0,0 @@ -[manifest] -name = xz - -# ubuntu's package causes watchman's tests to hang -[debs.not(distro=ubuntu)] -liblzma-dev - -[homebrew] -xz - -[rpms] -xz-devel - -[download] -url = https://tukaani.org/xz/xz-5.2.5.tar.gz -sha256 = f6f4910fd033078738bd82bfba4f49219d03b17eb0794eb91efbae419f4aba10 - -[build] -builder = autoconf -subdir = xz-5.2.5 diff --git a/build/fbcode_builder/manifests/yaml-cpp b/build/fbcode_builder/manifests/yaml-cpp deleted file mode 100644 index bffa540fe..000000000 --- a/build/fbcode_builder/manifests/yaml-cpp +++ /dev/null @@ -1,20 +0,0 @@ -[manifest] -name = yaml-cpp - -[download] -url = https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-0.6.2.tar.gz -sha256 = e4d8560e163c3d875fd5d9e5542b5fd5bec810febdcba61481fe5fc4e6b1fd05 - -[build.os=linux] -builder = cmake -subdir = yaml-cpp-yaml-cpp-0.6.2 - -[build.not(os=linux)] -builder = nop - -[dependencies] -boost -googletest - -[cmake.defines] -YAML_CPP_BUILD_TESTS=OFF diff --git a/build/fbcode_builder/manifests/zlib b/build/fbcode_builder/manifests/zlib deleted file mode 100644 index 9a7da1686..000000000 --- a/build/fbcode_builder/manifests/zlib +++ /dev/null @@ -1,28 +0,0 @@ -[manifest] -name = zlib - -[debs] -zlib1g-dev - -[homebrew] -zlib - -[rpms.not(distro=fedora)] -zlib-devel -zlib-static - -[rpms.distro=fedora] -zlib-ng-compat-devel -zlib-ng-compat-static - -[pps] -zlib - -[download] -url = https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz -sha256 = 9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23 - -[build] -builder = cmake -subdir = zlib-1.3.1 -patchfile = zlib_dont_build_more_than_needed.patch diff --git a/build/fbcode_builder/manifests/zlib-python b/build/fbcode_builder/manifests/zlib-python deleted file mode 100644 index d18997eb1..000000000 --- a/build/fbcode_builder/manifests/zlib-python +++ /dev/null @@ -1,34 +0,0 @@ -[manifest] -name = zlib-python - -[debs] -zlib1g-dev - -[homebrew] -zlib - -[rpms.not(distro=fedora)] -zlib-devel -zlib-static - -[rpms.distro=fedora] -zlib-ng-compat-devel -zlib-ng-compat-static - -[pps] -zlib - -[download] -url = https://zlib.net/zlib-1.3.1.tar.gz -sha256 = 9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23 - -[build] -builder = cmake -subdir = zlib-1.3.1 -patchfile = zlib_dont_build_more_than_needed.patch - -[build.not(os=linux)] -builder = nop - -[cmake.defines] -CMAKE_POSITION_INDEPENDENT_CODE=ON diff --git a/build/fbcode_builder/manifests/zstd b/build/fbcode_builder/manifests/zstd deleted file mode 100644 index 56bb64f7e..000000000 --- a/build/fbcode_builder/manifests/zstd +++ /dev/null @@ -1,36 +0,0 @@ -[manifest] -name = zstd - -[homebrew] -zstd - -# 18.04 zstd is too old -[debs.not(all(distro=ubuntu,distro_vers="18.04"))] -libclang-dev -libzstd-dev -zstd - -[rpms] -libzstd-devel -libzstd - -[pps] -zstd - -[download] -url = https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz -sha256 = 9c4396cc829cfae319a6e2615202e82aad41372073482fce286fac78646d3ee4 - -[build] -builder = cmake -subdir = zstd-1.5.5/build/cmake - -# The zstd cmake build explicitly sets the install name -# for the shared library in such a way that cmake discards -# the path to the library from the install_name, rendering -# the library non-resolvable during the build. The short -# term solution for this is just to link static on macos. -# -# And while we're at it, let's just always link statically. -[cmake.defines] -ZSTD_BUILD_SHARED = OFF diff --git a/build/fbcode_builder/manifests/zstd-python b/build/fbcode_builder/manifests/zstd-python deleted file mode 100644 index bb3ae56d8..000000000 --- a/build/fbcode_builder/manifests/zstd-python +++ /dev/null @@ -1,40 +0,0 @@ -[manifest] -name = zstd-python - -[homebrew] -zstd - -# 18.04 zstd is too old -[debs.not(all(distro=ubuntu,distro_vers="18.04"))] -libclang-dev -libzstd-dev -zstd - -[rpms] -libzstd-devel -libzstd - -[pps] -zstd - -[download] -url = https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz -sha256 = 9c4396cc829cfae319a6e2615202e82aad41372073482fce286fac78646d3ee4 - -[build] -builder = cmake -subdir = zstd-1.5.5/build/cmake - -[build.not(os=linux)] -builder = nop - -# The zstd cmake build explicitly sets the install name -# for the shared library in such a way that cmake discards -# the path to the library from the install_name, rendering -# the library non-resolvable during the build. The short -# term solution for this is just to link static on macos. -# -# And while we're at it, let's just always link statically. -[cmake.defines] -ZSTD_BUILD_SHARED = OFF -CMAKE_POSITION_INDEPENDENT_CODE=ON diff --git a/build/fbcode_builder/patches/boost_1_83_0.patch b/build/fbcode_builder/patches/boost_1_83_0.patch deleted file mode 100644 index 868a4a62d..000000000 --- a/build/fbcode_builder/patches/boost_1_83_0.patch +++ /dev/null @@ -1,29 +0,0 @@ -diff --git a/boost/serialization/strong_typedef.hpp b/boost/serialization/strong_typedef.hpp ---- a/boost/serialization/strong_typedef.hpp -+++ b/boost/serialization/strong_typedef.hpp -@@ -44,6 +44,7 @@ - operator const T&() const {return t;} \ - operator T&() {return t;} \ - bool operator==(const D& rhs) const {return t == rhs.t;} \ -+ bool operator==(const T& lhs) const {return t == lhs;} \ - bool operator<(const D& rhs) const {return t < rhs.t;} \ - }; - -diff --git a/tools/build/src/tools/msvc.jam b/tools/build/src/tools/msvc.jam ---- a/tools/build/src/tools/msvc.jam -+++ b/tools/build/src/tools/msvc.jam -@@ -1137,6 +1137,14 @@ - } - else - { -+ if [ MATCH "(14.4)" : $(version) ] -+ { -+ if $(.debug-configuration) -+ { -+ ECHO "notice: [generate-setup-cmd] $(version) is 14.4x" ; -+ } -+ parent = [ path.native [ path.join $(parent) "..\\..\\..\\..\\..\\Auxiliary\\Build" ] ] ; -+ } - if [ MATCH "(14.3)" : $(version) ] - { - if $(.debug-configuration) diff --git a/build/fbcode_builder/patches/iproute2_oss.patch b/build/fbcode_builder/patches/iproute2_oss.patch deleted file mode 100644 index 7c478afca..000000000 --- a/build/fbcode_builder/patches/iproute2_oss.patch +++ /dev/null @@ -1,36 +0,0 @@ -diff --git a/bridge/fdb.c b/bridge/fdb.c ---- a/bridge/fdb.c -+++ b/bridge/fdb.c -@@ -31,7 +31,7 @@ - - static unsigned int filter_index, filter_vlan, filter_state; - --json_writer_t *jw_global; -+static json_writer_t *jw_global; - - static void usage(void) - { -diff --git a/ip/ipmroute.c b/ip/ipmroute.c ---- a/ip/ipmroute.c -+++ b/ip/ipmroute.c -@@ -44,7 +44,7 @@ - exit(-1); - } - --struct rtfilter { -+static struct rtfilter { - int tb; - int af; - int iif; -diff --git a/ip/xfrm_monitor.c b/ip/xfrm_monitor.c ---- a/ip/xfrm_monitor.c -+++ b/ip/xfrm_monitor.c -@@ -34,7 +34,7 @@ - #include "ip_common.h" - - static void usage(void) __attribute__((noreturn)); --int listen_all_nsid; -+static int listen_all_nsid; - - static void usage(void) - { diff --git a/build/fbcode_builder/patches/libiberty_install_pic_lib.patch b/build/fbcode_builder/patches/libiberty_install_pic_lib.patch deleted file mode 100644 index 6346c3b30..000000000 --- a/build/fbcode_builder/patches/libiberty_install_pic_lib.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/Makefile.in b/Makefile.in -index b77a41c..cbe71fe 100644 ---- a/Makefile.in -+++ b/Makefile.in -@@ -389,7 +389,7 @@ MULTIOSDIR = `$(CC) $(CFLAGS) -print-multi-os-directory` - install_to_libdir: all - if test -n "${target_header_dir}"; then \ - ${mkinstalldirs} $(DESTDIR)$(libdir)/$(MULTIOSDIR); \ -- $(INSTALL_DATA) $(TARGETLIB) $(DESTDIR)$(libdir)/$(MULTIOSDIR)/$(TARGETLIB)n; \ -+ $(INSTALL_DATA) pic/$(TARGETLIB) $(DESTDIR)$(libdir)/$(MULTIOSDIR)/$(TARGETLIB)n; \ - ( cd $(DESTDIR)$(libdir)/$(MULTIOSDIR) ; chmod 644 $(TARGETLIB)n ;$(RANLIB) $(TARGETLIB)n ); \ - mv -f $(DESTDIR)$(libdir)/$(MULTIOSDIR)/$(TARGETLIB)n $(DESTDIR)$(libdir)/$(MULTIOSDIR)/$(TARGETLIB); \ - case "${target_header_dir}" in \ diff --git a/build/fbcode_builder/patches/zlib_dont_build_more_than_needed.patch b/build/fbcode_builder/patches/zlib_dont_build_more_than_needed.patch deleted file mode 100644 index f88df67e7..000000000 --- a/build/fbcode_builder/patches/zlib_dont_build_more_than_needed.patch +++ /dev/null @@ -1,34 +0,0 @@ -diff -Naur ../zlib-1.3.1/CMakeLists.txt ./CMakeLists.txt ---- ../zlib-1.3.1/CMakeLists.txt 2024-01-22 10:32:37.000000000 -0800 -+++ ./CMakeLists.txt 2024-01-23 13:14:09.870289968 -0800 -@@ -149,10 +149,8 @@ - set(ZLIB_DLL_SRCS ${CMAKE_CURRENT_BINARY_DIR}/zlib1rc.obj) - endif(MINGW) - --add_library(zlib SHARED ${ZLIB_SRCS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) -+add_library(zlib ${ZLIB_SRCS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) - target_include_directories(zlib PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) --add_library(zlibstatic STATIC ${ZLIB_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) --target_include_directories(zlibstatic PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) - set_target_properties(zlib PROPERTIES DEFINE_SYMBOL ZLIB_DLL) - set_target_properties(zlib PROPERTIES SOVERSION 1) - -@@ -169,7 +167,7 @@ - - if(UNIX) - # On unix-like platforms the library is almost always called libz -- set_target_properties(zlib zlibstatic PROPERTIES OUTPUT_NAME z) -+ set_target_properties(zlib PROPERTIES OUTPUT_NAME z) - if(NOT APPLE AND NOT(CMAKE_SYSTEM_NAME STREQUAL AIX)) - set_target_properties(zlib PROPERTIES LINK_FLAGS "-Wl,--version-script,\"${CMAKE_CURRENT_SOURCE_DIR}/zlib.map\"") - endif() -@@ -179,7 +177,7 @@ - endif() - - if(NOT SKIP_INSTALL_LIBRARIES AND NOT SKIP_INSTALL_ALL ) -- install(TARGETS zlib zlibstatic -+ install(TARGETS zlib - RUNTIME DESTINATION "${INSTALL_BIN_DIR}" - ARCHIVE DESTINATION "${INSTALL_LIB_DIR}" - LIBRARY DESTINATION "${INSTALL_LIB_DIR}" ) - \ No newline at end of file diff --git a/cinderx/.clang-tidy b/cinderx/.clang-tidy index 73246224e..529f3eddc 100644 --- a/cinderx/.clang-tidy +++ b/cinderx/.clang-tidy @@ -2,6 +2,15 @@ InheritParentConfig: true # Explaining why some of our options are set: # +# * Disabling enum value exhaustiveness checking. This lint fires even when +# there's a default label set. In CinderX we have some enums with many many +# values and we have many switch statements where we don't want or need to +# handle all possible cases explicitly, we want to push them through the +# default case. +# +# * `facebook-avoid-non-const-global-variables`: CPython C-extension APIs use +# non-const global type/spec objects, so this check is noisy here. +# # * Tracking nullable returns requires depending on folly, which we don't do. # It is also tedious as we would have to use this excessively due to Python # often using null to propagate exceptions. @@ -22,6 +31,8 @@ InheritParentConfig: true # to avoid this. Checks: ' +-clang-diagnostic-switch-enum, +-facebook-avoid-non-const-global-variables, -facebook-hte-BoostRegexRisky, -facebook-hte-GlibcRegexIsAwful, -facebook-hte-NullableReturn, diff --git a/cinderx/.gitignore b/cinderx/.gitignore index 206af5ffb..fffc43631 100644 --- a/cinderx/.gitignore +++ b/cinderx/.gitignore @@ -23,3 +23,5 @@ compile_commands.json *.a *.o *.so + +local_smoke_tests.md diff --git a/cinderx/AGENTS.md b/cinderx/AGENTS.md index f3ed061cd..542b599fe 100644 --- a/cinderx/AGENTS.md +++ b/cinderx/AGENTS.md @@ -3,9 +3,9 @@ This file provides guidance to AI coding agents when working with code in this directory. -If the file Internal/AGENTS.md exists, read this too. It contains details of -things which are relevant to Meta's internal developer environment but do not -apply for non Meta environments. +**Important**: If the file `Internal/AGENTS.md` exists, you MUST read it before +proceeding with any task. It contains additional context for internal +development environments. ## Overview @@ -26,6 +26,24 @@ select code to target different Python versions. Utilities are provided in `Common/` to abstract commonly used features which changed between Python versions. +## Different Target Architectures + +CinderX supports multiple architectures in its code generation backend +(e.g. x86-64, aarch64). The preference is to have all code compile under all +possible architectures, even on architectures where it is not used. Small blocks +of code can check the value of the `kBuildArch` constant with an `if constexpr` +statement. For entire functions, it's preferred to keep them unconditionally +defined, using the `[[maybe_unused]]` attribute to avoid unused code +warnings. Both of these cases assume that the underlying code can be compiled +across all architectures. If there is code that can only compile under a +specific hardware architecture, then that can be conditionally compiled by +checking preprocessor defines like `CINDER_X86_64` and `CINDER_AARCH64`. + +One specific case to highlight is struct/class fields that are only used on +specific architectures. If they are part of singletons then it's fine to define +them always even if they are unused, but otherwise they should be conditionally +compiled via the preprocessor defines, to save on memory usage. + ## Non-public Python APIs Where possible CinderX tries to use public Python APIs. When not @@ -92,7 +110,27 @@ If the instruction needs to call a custom runtime helper function: - Declare it in **Jit/jit_rt.h** - Implement it in **Jit/jit_rt.cpp** +## Handling JIT compile-time errors + +If an error is hit when JIT-compiling a Python function, the preference is to +raise a C++ exception. This will unwind the stack and silently fail the compile, +causing the Python function to return back to the interpreter as usual. + +The `JIT_THROW` and `JIT_THROW_IF` macros make it easy to raise an exception +that is tagged with the offending file and line number, to make debugging +easier. These are the preferred tool for handling irrecoverable JIT-compilation +errors. + +The `JIT_ABORT` and `JIT_CHECK` macros are similar but will crash the entire +process, which is usually undesirable. They should be used sparingly, and in +very restricted scenarios where throwing a C++ exception does not make sense. + +The `JIT_DCHECK` macro is intended for invariants that we'd like to enforce, but +cannot do so in production builds because they lie in performance-sensitive code +paths (e.g. the function vectorcall entry point that we install). + ## Investigating JIT failures + If you're investigating a JIT issue you may want to isolate the issue to a single function. You can use `cinderx.jit.force_compile` to compile an individual function if you suspect that a specific function is problematic. @@ -100,3 +138,7 @@ individual function if you suspect that a specific function is problematic. If you run the test with PYTHONJITDUMPASM=1 you can see the assembly dumped along with the HIR to understand what the compiled code looks like and what the underlying issue is. + +## Code style + +See `Internal/docs/style.md`. diff --git a/cinderx/CachedProperties/cached_properties.c b/cinderx/CachedProperties/cached_properties.c index f42efa646..50b40dc10 100644 --- a/cinderx/CachedProperties/cached_properties.c +++ b/cinderx/CachedProperties/cached_properties.c @@ -46,6 +46,7 @@ cached_classproperty_new(PyTypeObject* type, PyObject* args, PyObject* kwds) { PyObject* name; if (PyFunction_Check(func)) { name = ((PyFunctionObject*)func)->func_name; + Py_INCREF(name); } else { name = PyObject_GetAttrString(func, "__name__"); if (name == NULL) { @@ -56,7 +57,6 @@ cached_classproperty_new(PyTypeObject* type, PyObject* args, PyObject* kwds) { descr->func = func; descr->name = name; Py_INCREF(func); - Py_INCREF(name); } return (PyObject*)descr; } @@ -542,7 +542,6 @@ PyTypeObject PyCachedProperty_Type = { PyTypeObject PyCachedPropertyWithDescr_Type = { PyVarObject_HEAD_INIT(NULL, 0).tp_name = "cached_property_with_descr", - .tp_base = &PyCachedProperty_Type, .tp_basicsize = sizeof(PyCachedPropertyDescrObject), .tp_dealloc = (destructor)cached_property_dealloc, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, @@ -627,7 +626,6 @@ static inline int import_async_lazy_value() { } DEFINE_STATIC_STRING(AsyncLazyValue); _AsyncLazyValue_Type = PyObject_GetAttr(module, s_AsyncLazyValue); -#if PY_VERSION_HEX >= 0x030C0000 // _asyncio can be overridden with a version that has AsyncLazyValue, // if it's not there fallback to CinderX's builtin version. if (_AsyncLazyValue_Type == NULL) { @@ -639,7 +637,6 @@ static inline int import_async_lazy_value() { } _AsyncLazyValue_Type = PyObject_GetAttr(module, s_AsyncLazyValue); } -#endif Py_DECREF(module); if (_AsyncLazyValue_Type == NULL) { return -1; diff --git a/cinderx/CachedProperties/cached_properties.h b/cinderx/CachedProperties/cached_properties.h index 6bd9132f5..6eb17431a 100644 --- a/cinderx/CachedProperties/cached_properties.h +++ b/cinderx/CachedProperties/cached_properties.h @@ -1,7 +1,6 @@ // Copyright (c) Meta Platforms, Inc. and affiliates. -#ifndef Py_CACHED_PROPERTIES_H -#define Py_CACHED_PROPERTIES_H +#pragma once #include "cinderx/python.h" @@ -30,11 +29,15 @@ typedef struct { } PyAsyncCachedClassPropertyDescrObject; /* end fb T82701047 */ +#ifdef __cplusplus +extern "C" { +#endif extern PyTypeObject PyAsyncCachedPropertyWithDescr_Type; extern PyType_Spec _PyCachedClassProperty_TypeSpec; /* fb t46346203 */ extern PyTypeObject PyCachedProperty_Type; /* fb T46346203 */ extern PyTypeObject PyCachedPropertyWithDescr_Type; /* fb T46346203 */ extern PyTypeObject PyAsyncCachedProperty_Type; /* fb T82701047 */ extern PyTypeObject PyAsyncCachedClassProperty_Type; /* fb T82701047 */ - -#endif /* !Py_CACHED_PROPERTIES_H */ +#ifdef __cplusplus +} +#endif diff --git a/cinderx/Common/aligned_memory.h b/cinderx/Common/aligned_memory.h new file mode 100644 index 000000000..79ccae8a6 --- /dev/null +++ b/cinderx/Common/aligned_memory.h @@ -0,0 +1,50 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#pragma once + +#include "cinderx/Common/log.h" + +#include +#include +#include + +#ifdef WIN32 +#include +#endif + +namespace cinderx { + +struct AlignedMemoryDeleter { + void operator()(void* ptr) const { +#ifdef WIN32 + _aligned_free(ptr); +#else + std::free(ptr); +#endif + } +}; + +template +class AlignedMemory { + public: + AlignedMemory(size_t size, size_t alignment) { + void* ptr; +#ifdef WIN32 + ptr = _aligned_malloc(size, alignment); + JIT_CHECK(ptr != nullptr, "Failed to allocate {} bytes", size); +#else + int result = posix_memalign(&ptr, alignment, size); + JIT_CHECK(result == 0, "Failed to allocate {} bytes", size); +#endif + ptr_.reset(static_cast(ptr)); + } + + T* get() const { + return ptr_.get(); + } + + private: + std::unique_ptr ptr_; +}; + +} // namespace cinderx diff --git a/cinderx/Common/audit.cpp b/cinderx/Common/audit.cpp index 81f02e79a..0925e3147 100644 --- a/cinderx/Common/audit.cpp +++ b/cinderx/Common/audit.cpp @@ -23,13 +23,7 @@ bool installAuditHook(Py_AuditHookFunction func, void* userData) { return true; } #endif - _Py_AuditHookEntry* audit_hook_head = -#if PY_VERSION_HEX < 0x030C0000 - runtime->audit_hook_head -#else - runtime->audit_hooks.head -#endif - ; + _Py_AuditHookEntry* audit_hook_head = runtime->audit_hooks.head; // Verify that the hook was actually installed. for (_Py_AuditHookEntry* e = audit_hook_head; e != nullptr; e = e->next) { diff --git a/cinderx/Common/bump_arena.cpp b/cinderx/Common/bump_arena.cpp new file mode 100644 index 000000000..6c1a0f22a --- /dev/null +++ b/cinderx/Common/bump_arena.cpp @@ -0,0 +1,64 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#include "cinderx/Common/bump_arena.h" + +#include "cinderx/Common/util.h" + +#include +#include + +namespace cinderx { + +namespace { + +constexpr size_t kAlign = size_t{kPageSize}; +constexpr size_t kMaxBlockSize = size_t{kMiB}; + +} // namespace + +BumpArena::~BumpArena() { + for (auto it = destructors_.rbegin(); it != destructors_.rend(); ++it) { + it->destroy(it->obj); + } +} + +void* BumpArena::allocateBytes(size_t size, size_t alignment) { + JIT_DCHECK(isPowerOfTwo(alignment), "Alignment must be a power of 2"); + + Block* block; + size_t offset; + + if (blocks_.empty()) { + block = &addBlock(size, alignment); + offset = 0; + } else { + block = &blocks_.back(); + + const auto base = reinterpret_cast(block->base.get()); + offset = roundUp(base + block->fill, alignment) - base; + + if (offset > block->size || size > block->size - offset) { + block = &addBlock(size, alignment); + offset = 0; + + JIT_DCHECK(offset <= block->size, "BumpArena block alignment too large"); + JIT_DCHECK(size <= block->size - offset, "BumpArena block too small"); + } + } + + void* ptr = block->base.get() + offset; + block->fill = offset + size; + return ptr; +} + +BumpArena::Block& BumpArena::addBlock(size_t min_size, size_t alignment) { + const size_t size = roundUp(std::max(min_size, next_block_size_), kAlign); + next_block_size_ = std::min(next_block_size_ * 2, kMaxBlockSize); + + Block& block = blocks_.emplace_back( + Block{AlignedMemory{size, std::max(alignment, kAlign)}, 0, size}); + + return block; +} + +} // namespace cinderx diff --git a/cinderx/Common/bump_arena.h b/cinderx/Common/bump_arena.h new file mode 100644 index 000000000..c72f57546 --- /dev/null +++ b/cinderx/Common/bump_arena.h @@ -0,0 +1,75 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#pragma once + +#include "cinderx/Common/aligned_memory.h" +#include "cinderx/Common/slab_arena.h" + +#include +#include +#include +#include +#include +#include + +namespace cinderx { + +// A mixed-type bump allocator that keeps allocated addresses stable and +// destroys non-trivially-destructible objects when the arena is destroyed. +class BumpArena { + public: + BumpArena() = default; + ~BumpArena(); + + BumpArena(const BumpArena&) = delete; + BumpArena& operator=(const BumpArena&) = delete; + BumpArena(BumpArena&&) = delete; + BumpArena& operator=(BumpArena&&) = delete; + + template < + typename T, + typename SizeTrait = ObjectSizeTrait, + typename... Args> + T* allocate(Args&&... args) { + std::lock_guard guard{mutex_}; + + const size_t size = SizeTrait::size(); + JIT_CHECK(size >= sizeof(T), "SizeTrait must allocate enough space"); + + void* mem = allocateBytes(size, alignof(T)); + T* obj = new (mem) T(std::forward(args)...); + + /* If there is a destructor, we are going to track it here so that when the + * whole arena is freed we can call it. */ + if constexpr (!std::is_trivially_destructible_v) { + static_assert(std::is_nothrow_destructible_v); + destructors_.push_back(Destructor{obj, [](void* ptr) noexcept { + std::destroy_at(static_cast(ptr)); + }}); + } + + return obj; + } + + private: + struct Block { + AlignedMemory base; + size_t fill{0}; + size_t size{0}; + }; + + struct Destructor { + void* obj; + void (*destroy)(void*) noexcept; + }; + + void* allocateBytes(size_t size, size_t alignment); + Block& addBlock(size_t min_size, size_t alignment); + + std::vector blocks_; + std::vector destructors_; + size_t next_block_size_{size_t{kPageSize}}; + std::mutex mutex_; +}; + +} // namespace cinderx diff --git a/cinderx/Common/code.cpp b/cinderx/Common/code.cpp index 6ad137ff6..c6268bbf6 100644 --- a/cinderx/Common/code.cpp +++ b/cinderx/Common/code.cpp @@ -9,24 +9,36 @@ #include "cinderx/Common/util.h" #include "cinderx/Interpreter/cinder_opcode.h" #include "cinderx/UpstreamBorrow/borrowed.h" // @donotremove +#include "cinderx/module_state.h" +#ifdef ENABLE_ZLIB #include - -#if PY_VERSION_HEX >= 0x030C0000 +#endif #include "cpython/code.h" -#endif - namespace { -// Index used for fetching code object extra data. -Py_ssize_t code_extra_index = -1; - -} // namespace +// Read the existing CodeExtra for a code object without allocating one, unlike +// codeExtra(). Returns nullptr if none has been created yet. +CodeExtra* codeExtraIfPresent(BorrowedRef code) { + auto state = cinderx::getModuleState(); + if (state == nullptr) { + return nullptr; + } + Py_ssize_t extra_index = state->code_extra_index; + if (extra_index == -1) { + return nullptr; + } + void* data_ptr = nullptr; + if (PyUnstable_Code_GetExtra(code.getObj(), extra_index, &data_ptr) < 0) { + PyErr_Clear(); + return nullptr; + } + return reinterpret_cast(data_ptr); +} -namespace jit { -static std::string fullnameImpl(PyObject* module, PyObject* qualname) { +std::string fullnameImpl(PyObject* module, PyObject* qualname) { auto safe_str = [](BorrowedRef<> str) { if (str == nullptr || !PyUnicode_Check(str)) { return ""; @@ -36,6 +48,10 @@ static std::string fullnameImpl(PyObject* module, PyObject* qualname) { return fmt::format("{}:{}", safe_str(module), safe_str(qualname)); } +} // namespace + +namespace cinderx { + std::string codeFullname( BorrowedRef module, BorrowedRef code) { @@ -52,9 +68,10 @@ PyObject* getVarnameTuple(BorrowedRef code, int* idx) { } *idx -= code->co_nlocals; - auto ncellvars = PyTuple_GET_SIZE(PyCode_GetCellvars(code)); + auto cellvars = Ref<>::steal(PyCode_GetCellvars(code)); + auto ncellvars = PyTuple_GET_SIZE(cellvars.get()); if (*idx < ncellvars) { - return PyCode_GetCellvars(code); + return cellvars.release(); } *idx -= ncellvars; @@ -62,17 +79,13 @@ PyObject* getVarnameTuple(BorrowedRef code, int* idx) { } PyObject* getVarname(BorrowedRef code, int idx) { -#if PY_VERSION_HEX >= 0x030C0000 return PyTuple_GET_ITEM(code->co_localsplusnames, idx); -#else - PyObject* tuple = getVarnameTuple(code, &idx); - return PyTuple_GET_ITEM(tuple, idx); -#endif } uint32_t hashBytecode(BorrowedRef code) { + auto bc = Ref<>::steal(PyCode_GetCode(code)); +#ifdef ENABLE_ZLIB uint32_t crc = crc32(0, nullptr, 0); - PyObject* bc = PyCode_GetCode(code); if (!PyBytes_Check(bc)) { return crc; } @@ -84,6 +97,9 @@ uint32_t hashBytecode(BorrowedRef code) { } return crc32(crc, reinterpret_cast(buffer), len); +#else + return PyObject_Hash(bc); +#endif } std::string codeQualname(BorrowedRef code) { @@ -96,7 +112,7 @@ std::string codeQualname(BorrowedRef code) { return ""; } -} // namespace jit +} // namespace cinderx extern "C" { @@ -108,41 +124,24 @@ const char* codeName(PyCodeObject* code) { } _Py_CODEUNIT* codeUnit(PyCodeObject* code) { -#if PY_VERSION_HEX >= 0x030C0000 return _PyCode_CODE(code); -#else - PyObject* bytes_obj = PyCode_GetCode(code); - JIT_DCHECK( - PyBytes_CheckExact(bytes_obj), - "Code object must have its instructions stored as a byte string"); - return (_Py_CODEUNIT*)PyBytes_AS_STRING(PyCode_GetCode(code)); -#endif } size_t countIndices(PyCodeObject* code) { -#if PY_VERSION_HEX >= 0x030C0000 // PyCode_GetCode can allocate to create a copy of the de-opted code // which we don't need just to determine the number of indices. return _PyCode_NBYTES(code) / sizeof(_Py_CODEUNIT); -#else - return PyBytes_GET_SIZE(PyCode_GetCode(code)) / sizeof(_Py_CODEUNIT); -#endif } int unspecialize(int opcode) { -#if PY_VERSION_HEX >= 0x030C0000 // The deopt table has size 256, and pseudo-opcodes and stubs are by // definition unspecialized already. return (opcode >= 0 && opcode <= 255) ? _CiOpcode_Deopt[opcode] : opcode; -#else - return opcode; -#endif } int uninstrument(PyCodeObject* code, int index) { int opcode = _Py_OPCODE(codeUnit(code)[index]); -#if PY_VERSION_HEX >= 0x030C0000 // Check if there's an equivalent opcode without instrumentation. uint8_t base_opcode = Cix_DEINSTRUMENT(static_cast(opcode)); if (base_opcode != 0) { @@ -157,7 +156,6 @@ int uninstrument(PyCodeObject* code, int index) { if (opcode == INSTRUMENTED_LINE) { return Cix_GetOriginalOpcode(code->_co_monitoring->lines, index); } -#endif return opcode; } @@ -172,58 +170,50 @@ const char* opcodeName(int opcode) { return name != nullptr ? name : ""; } -Py_ssize_t inlineCacheSize( - [[maybe_unused]] PyCodeObject* code, - [[maybe_unused]] int index) { -#if PY_VERSION_HEX >= 0x030C0000 +Py_ssize_t inlineCacheSize(PyCodeObject* code, int index) { return _CiOpcode_Caches[unspecialize(uninstrument(code, index))]; -#else - return 0; -#endif } int loadAttrIndex(int oparg) { - if constexpr (PY_VERSION_HEX >= 0x030C0000) { - return oparg >> 1; - } - return oparg; + return oparg >> 1; } int loadGlobalIndex(int oparg) { - if constexpr (PY_VERSION_HEX >= 0x030B0000) { - return oparg >> 1; - } - return oparg; + return oparg >> 1; } void initCodeExtraIndex() { - if constexpr (!USE_CODE_EXTRA) { - return; - } + auto state = cinderx::getModuleState(); + JIT_CHECK( + state != nullptr, + "Trying to initialize code extra index but there's no module state"); JIT_CHECK( - code_extra_index == -1, + state->code_extra_index == -1, "Cannot re-initialize code extra index without finalizing it first"); - code_extra_index = PyUnstable_Eval_RequestCodeExtraIndex(PyMem_Free); + state->code_extra_index = PyUnstable_Eval_RequestCodeExtraIndex(PyMem_Free); } void finiCodeExtraIndex() { - if constexpr (!USE_CODE_EXTRA) { - return; - } + auto state = cinderx::getModuleState(); + JIT_CHECK( + state != nullptr, + "Trying to finalize code extra index but there's no module state"); JIT_CHECK( - code_extra_index != -1, + state->code_extra_index != -1, "Cannot finalize code extra index without initializing it first"); - code_extra_index = -1; + state->code_extra_index = -1; } CodeExtra* codeExtra(PyCodeObject* code) { - if constexpr (!USE_CODE_EXTRA) { + auto* state = cinderx::getModuleState(); + // On shutdown the module state becomes inaccessible. + if (state == nullptr) { return nullptr; } - - if (code_extra_index == -1) { + Py_ssize_t extra_index = state->code_extra_index; + if (extra_index == -1) { return nullptr; } @@ -231,12 +221,12 @@ CodeExtra* codeExtra(PyCodeObject* code) { // Lock the code object to prevent concurrent get-or-create races under // FT-Python. Under GIL builds this is a no-op. - jit::CriticalSectionGuard guard(code_obj); + cinderx::CriticalSectionGuard guard(code_obj); void* data_ptr = nullptr; - if (PyUnstable_Code_GetExtra(code_obj, code_extra_index, &data_ptr) < 0) { + if (PyUnstable_Code_GetExtra(code_obj, extra_index, &data_ptr) < 0) { JIT_LOG("Failed to get code extra data for {}", codeName(code)); - jit::printPythonException(); + cinderx::printPythonException(); PyErr_Clear(); return nullptr; } @@ -249,9 +239,9 @@ CodeExtra* codeExtra(PyCodeObject* code) { return nullptr; } - if (PyUnstable_Code_SetExtra(code_obj, code_extra_index, extra) < 0) { + if (PyUnstable_Code_SetExtra(code_obj, extra_index, extra) < 0) { JIT_LOG("Failed to set code extra data for {}", codeName(code)); - jit::printPythonException(); + cinderx::printPythonException(); PyErr_Clear(); PyMem_Free(extra); return nullptr; @@ -260,32 +250,27 @@ CodeExtra* codeExtra(PyCodeObject* code) { return extra; } +size_t codeCallCount(PyCodeObject* code) { + // Don't allocate the CodeExtra if it doesn't exist, to allow for running this + // from within the multithreaded compile pipeline where the GIL doesn't exist. + CodeExtra* extra = codeExtraIfPresent(code); + return extra != nullptr ? Ci_code_extra_get_calls(extra) : 0; +} + int numLocals(PyCodeObject* code) { return code->co_nlocals; } int numCellvars(PyCodeObject* code) { -#if PY_VERSION_HEX >= 0x030B0000 return code->co_ncellvars; -#else - return PyTuple_GET_SIZE(PyCode_GetCellvars(code)); -#endif } int numFreevars(PyCodeObject* code) { -#if PY_VERSION_HEX >= 0x030B0000 return code->co_nfreevars; -#else - return PyTuple_GET_SIZE(PyCode_GetFreevars(code)); -#endif } int numLocalsplus(PyCodeObject* code) { -#if PY_VERSION_HEX >= 0x030B0000 return code->co_nlocalsplus; -#else - return numLocals(code) + numCellvars(code) + numFreevars(code); -#endif } } // extern "C" diff --git a/cinderx/Common/code.h b/cinderx/Common/code.h index 3e47b2466..426cea8d4 100644 --- a/cinderx/Common/code.h +++ b/cinderx/Common/code.h @@ -17,72 +17,6 @@ extern "C" { #endif -// The following PyCodeObject functions were added in 3.11. -#if PY_VERSION_HEX < 0x030B0000 - -static inline PyObject* PyCode_GetCode(PyCodeObject* code) { - return code->co_code; -} - -static inline PyObject* PyCode_GetVarnames(PyCodeObject* code) { - return code->co_varnames; -} - -static inline PyObject* PyCode_GetCellvars(PyCodeObject* code) { - return code->co_cellvars; -} - -static inline PyObject* PyCode_GetFreevars(PyCodeObject* code) { - return code->co_freevars; -} - -static inline PyCodeObject* PyUnstable_Code_New( - int argcount, - int kwonlyargcount, - int nlocals, - int stacksize, - int flags, - PyObject* code, - PyObject* consts, - PyObject* names, - PyObject* varnames, - PyObject* freevars, - PyObject* cellvars, - PyObject* filename, - PyObject* name, - PyObject* Py_UNUSED(qualname), - int firstlineno, - PyObject* linetable, - PyObject* Py_UNUSED(exceptiontable)) { - return PyCode_New( - argcount, - kwonlyargcount, - nlocals, - stacksize, - flags, - code, - consts, - names, - varnames, - freevars, - cellvars, - filename, - name, - firstlineno, - linetable); -} - -#endif - -#if PY_VERSION_HEX < 0x030C0000 - -// Renamed in 3.12. -#define PyUnstable_Eval_RequestCodeExtraIndex _PyEval_RequestCodeExtraIndex -#define PyUnstable_Code_GetExtra _PyCode_GetExtra -#define PyUnstable_Code_SetExtra _PyCode_SetExtra - -#endif - // Gets the qualified name of the code object or "" if it's not set. const char* codeName(PyCodeObject* code); @@ -115,9 +49,6 @@ int loadAttrIndex(int oparg); // Get the name index from a LOAD_GLOBAL's oparg. int loadGlobalIndex(int oparg); -// Before 3.12, Cinder relies on Shadowcode's call count tracking. -#define USE_CODE_EXTRA (PY_VERSION_HEX >= 0x030C0000) - // Initialize and finalize the index of the extra data Cinder attaches onto code // objects. void initCodeExtraIndex(); @@ -128,6 +59,10 @@ void finiCodeExtraIndex(); // Python error set. CodeExtra* codeExtra(PyCodeObject* code); +// Get the number of times a code object has been called by the interpreter. +// Calls to JIT-compiled code objects are currently uncounted. +size_t codeCallCount(PyCodeObject* code); + // Count the various frame variables that a code object will use. int numLocals(PyCodeObject* code); int numCellvars(PyCodeObject* code); @@ -140,7 +75,7 @@ int numLocalsplus(PyCodeObject* code); uint8_t Cix_GetOriginalOpcode( _PyCoLineInstrumentationData* line_data, int index); -#elif PY_VERSION_HEX >= 0x030C0000 +#else static inline uint8_t Cix_GetOriginalOpcode( _PyCoLineInstrumentationData* line_data, int index) { @@ -155,7 +90,7 @@ static inline uint8_t Cix_GetOriginalOpcode( #include -namespace jit { +namespace cinderx { std::string codeFullname( BorrowedRef module, @@ -163,20 +98,21 @@ std::string codeFullname( std::string funcFullname(BorrowedRef func); // Given a code object and an index into f_localsplus, compute which of -// code->co_varnames, code->cellvars, or code->freevars contains the name of -// the variable. Return that tuple and adjust idx as needed. +// code->co_varnames, code->cellvars, or code->freevars contains the name of the +// variable. Return a new reference to that tuple and adjust idx as needed. PyObject* getVarnameTuple(BorrowedRef code, int* idx); // Similar to getVarnameTuple, but return the name itself rather than the // containing tuple. PyObject* getVarname(BorrowedRef code, int idx); +// Return a crc32 checksum of the bytecode for the given code object. uint32_t hashBytecode(BorrowedRef code); // Return the qualname of the given code object, falling back to its name or // "" if not set. std::string codeQualname(BorrowedRef code); -} // namespace jit +} // namespace cinderx #endif diff --git a/cinderx/Common/compiler.h b/cinderx/Common/compiler.h new file mode 100644 index 000000000..d6632bfa2 --- /dev/null +++ b/cinderx/Common/compiler.h @@ -0,0 +1,26 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#pragma once + +// This file defines macros that allow use of compiler-specific features in a +// portable way. + +#ifndef __has_attribute +#define __has_attribute(x) 0 +#endif + +#if __has_attribute(noinline) +#define CINDERX_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) +#define CINDERX_NOINLINE __declspec(noinline) +#else +#define CINDERX_NOINLINE +#endif + +#if __has_attribute(always_inline) +#define CINDERX_ALWAYS_INLINE inline __attribute__((always_inline)) +#elif defined(_MSC_VER) +#define CINDERX_ALWAYS_INLINE __forceinline +#else +#define CINDERX_ALWAYS_INLINE inline +#endif diff --git a/cinderx/Jit/containers.h b/cinderx/Common/containers.h similarity index 99% rename from cinderx/Jit/containers.h rename to cinderx/Common/containers.h index 0d2abbb42..e21e0aedc 100644 --- a/cinderx/Jit/containers.h +++ b/cinderx/Common/containers.h @@ -33,7 +33,7 @@ #include #endif -namespace jit { +namespace cinderx { #define SET_TEMPLATE_PARAMS #define SET_ORDERED_TEMPLATE_PARAMS @@ -158,4 +158,4 @@ MAP_ORDERED_TEMPLATE_ARGS using OrderedMultimap = #undef MAP_TEMPLATE_ARGS #undef MAP_ORDERED_TEMPLATE_ARGS -} // namespace jit +} // namespace cinderx diff --git a/cinderx/Common/define.h b/cinderx/Common/define.h new file mode 100644 index 000000000..011611dbd --- /dev/null +++ b/cinderx/Common/define.h @@ -0,0 +1,103 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +// A collection of preprocessor defines converted into `constexpr` values that +// can be used unconditionally. +// +// This header should never depend on any other file, except for +// cinderx/python.h. + +#pragma once + +#include "cinderx/python.h" + +#if defined(__has_feature) +#define CINDER_HAS_FEATURE(x) __has_feature(x) +#else +#define CINDER_HAS_FEATURE(x) 0 +#endif + +#if defined(__SANITIZE_THREAD__) || CINDER_HAS_FEATURE(thread_sanitizer) +#define CINDER_TSAN_ENABLED 1 +#else +#define CINDER_TSAN_ENABLED 0 +#endif + +namespace cinderx { + +// Whether CinderX is being built with a debug build configuration. +constexpr bool kDebug = +#ifdef NDEBUG + false; +#else + true; +#endif + +// Whether the Python runtime was built with a debug build configuration. +constexpr bool kPyDebug = +#ifdef Py_DEBUG + true; +#else + false; +#endif + +constexpr bool kPyRefDebug = +#ifdef Py_REF_DEBUG + true; +#else + false; +#endif + +// True when CinderX is built against a free-threaded (Py_GIL_DISABLED) Python. +// +// When false, code can assume the GIL is held. When true, it cannot, the GIL +// might still be held at any given moment but that's no longer guaranteed. +constexpr bool kFreeThreadedBuild = +#ifdef Py_GIL_DISABLED + true; +#else + false; +#endif + +// True when CinderX is built for the prefork (fork-and-exec) process model, +// i.e. with the ENABLE_PREFORK_MODEL build flag. In this mode some behaviors +// that would otherwise be runtime options are forced on at compile time -- e.g. +// JIT-compiled functions are always immortalized, avoiding refcount churn that +// would otherwise be copied-on-write across forked worker processes. +// +// Prefer branching on this constexpr over #ifdef ENABLE_PREFORK_MODEL so the +// guarded code still gets type-checked in every build configuration. +constexpr bool kPreforkModel = +#ifdef ENABLE_PREFORK_MODEL + true; +#else + false; +#endif + +// The CPU architecture targeted by the current build. +enum class Arch { + kX86_64, + kAarch64, + kUnknown, +}; + +// This macro is a marker for places that need platform-specific code. +#define CINDER_UNSUPPORTED + +#if defined(__x86_64__) || defined(_M_AMD64) + +#define CINDER_X86_64 +constexpr Arch kBuildArch = Arch::kX86_64; + +#elif defined(__aarch64__) + +#define CINDER_AARCH64 +constexpr Arch kBuildArch = Arch::kAarch64; + +#else + +#define CINDER_UNKNOWN +constexpr Arch kBuildArch = Arch::kUnknown; + +#endif + +} // namespace cinderx diff --git a/cinderx/Common/dict.h b/cinderx/Common/dict.h index a7a858fc7..8c8618ebf 100644 --- a/cinderx/Common/dict.h +++ b/cinderx/Common/dict.h @@ -4,9 +4,11 @@ #include "cinderx/python.h" -#if PY_VERSION_HEX >= 0x030C0000 // This needs to come before borrowed.h #include "pycore_dict.h" + +#ifdef __cplusplus +#include "cinderx/Common/ref.h" #endif #include "cinderx/UpstreamBorrow/borrowed.h" @@ -17,31 +19,15 @@ extern "C" { #endif -#if PY_VERSION_HEX < 0x030C0000 - -#include "Objects/dict-common.h" // @donotremove -#define DICT_VALUES(dict) dict->ma_values - -#else - #define DICT_VALUES(dict) dict->ma_values->values #include "internal/pycore_dict.h" -#endif - static inline PyObject* getBorrowedTypeDict(PyTypeObject* self) { -#if PY_VERSION_HEX >= 0x030C0000 return _PyType_GetDict(self); -#else - assert(self->tp_dict != NULL); - return self->tp_dict; -#endif } -#if PY_VERSION_HEX >= 0x030C0000 #define _PyDict_NotifyEvent(EVENT, MP, KEY, VAL) \ _PyDict_NotifyEvent(_PyInterpreterState_GET(), (EVENT), (MP), (KEY), (VAL)) -#endif // Check if a dictionary is guaranteed to only contain unicode/string keys. // @@ -51,11 +37,7 @@ static inline PyObject* getBorrowedTypeDict(PyTypeObject* self) { static inline bool hasOnlyUnicodeKeys(PyObject* dict) { assert(PyDict_Check(dict)); -#if PY_VERSION_HEX >= 0x030C0000 return DK_IS_UNICODE(((PyDictObject*)dict)->ma_keys); -#else - return _PyDict_HasOnlyUnicodeKeys(dict); -#endif } static inline Py_ssize_t getDictKeysIndex( @@ -64,7 +46,6 @@ static inline Py_ssize_t getDictKeysIndex( #if PY_VERSION_HEX >= 0x030E0000 return _PyDictKeys_StringLookupSplit(keys, name); #endif -#if PY_VERSION_HEX >= 0x030C0000 for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { PyDictUnicodeEntry* ep = &DK_UNICODE_ENTRIES(keys)[i]; if (PyUnicode_Compare(name, ep->me_key) == 0) { @@ -72,14 +53,10 @@ static inline Py_ssize_t getDictKeysIndex( } } return -1; -#else - return _PyDictKeys_GetSplitIndex(keys, name); -#endif } // We can't borrow this from CPython because it exists but is not // exported, and therefore borrowing it duplicates the symbol. -#if PY_VERSION_HEX >= 0x030C0000 static inline uint32_t dictGetKeysVersion( PyInterpreterState* interp, PyDictKeysObject* dictkeys) { @@ -93,7 +70,6 @@ static inline uint32_t dictGetKeysVersion( dictkeys->dk_version = v; return v; } -#endif #if PY_VERSION_HEX >= 0x030E0000 typedef uint32_t ci_dict_version_tag_t; @@ -110,3 +86,22 @@ static inline ci_dict_version_tag_t Ci_DictVersionTag(PyDictObject* dict) { #ifdef __cplusplus } // extern "C" #endif + +#ifdef __cplusplus + +inline Ref<> getDictRef(PyObject* dict, PyObject* key) { +#if PY_VERSION_HEX >= 0x030E0000 + PyObject* res; + if (PyDict_GetItemRef(dict, key, &res) > 0) { + return Ref<>::steal(res); + } +#else + PyObject* res = PyDict_GetItemWithError(dict, key); + if (res != nullptr) { + return Ref<>::create(res); + } +#endif + return nullptr; +} + +#endif diff --git a/cinderx/Common/extra-py-flags.h b/cinderx/Common/extra-py-flags.h index 20c6767bf..d8912208d 100644 --- a/cinderx/Common/extra-py-flags.h +++ b/cinderx/Common/extra-py-flags.h @@ -19,7 +19,7 @@ #define Ci_Py_TPFLAGS_IS_STATICALLY_DEFINED (1UL << 21) -#elif PY_VERSION_HEX >= 0x030C0000 +#else // Lowest bit is unused #define Ci_Py_TPFLAGS_IS_STATICALLY_DEFINED (1UL << 2) diff --git a/cinderx/Common/fork_support.cpp b/cinderx/Common/fork_support.cpp new file mode 100644 index 000000000..734c9f684 --- /dev/null +++ b/cinderx/Common/fork_support.cpp @@ -0,0 +1,45 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#include "cinderx/Common/fork_support.h" + +#include "cinderx/Common/define.h" + +#if CINDER_TSAN_ENABLED +#include +#endif + +#include + +namespace cinderx { + +void resetMutexAfterFork(std::mutex& mutex) { +#if CINDER_TSAN_ENABLED + void* native_mutex = mutex.native_handle(); + __tsan_mutex_pre_unlock(native_mutex, 0); + __tsan_mutex_post_unlock(native_mutex, 0); + __tsan_mutex_destroy(native_mutex, 0); +#endif + + new (&mutex) std::mutex{}; + +#if CINDER_TSAN_ENABLED + __tsan_mutex_create(mutex.native_handle(), 0); +#endif +} + +void resetMutexAfterFork(std::recursive_mutex& mutex) { +#if CINDER_TSAN_ENABLED + void* native_mutex = mutex.native_handle(); + __tsan_mutex_pre_unlock(native_mutex, __tsan_mutex_recursive_unlock); + __tsan_mutex_post_unlock(native_mutex, 0); + __tsan_mutex_destroy(native_mutex, 0); +#endif + + new (&mutex) std::recursive_mutex{}; + +#if CINDER_TSAN_ENABLED + __tsan_mutex_create(mutex.native_handle(), __tsan_mutex_write_reentrant); +#endif +} + +} // namespace cinderx diff --git a/cinderx/Common/fork_support.h b/cinderx/Common/fork_support.h new file mode 100644 index 000000000..50ef07df2 --- /dev/null +++ b/cinderx/Common/fork_support.h @@ -0,0 +1,14 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#pragma once + +#include + +namespace cinderx { + +// Replace a mutex locked by an atfork prepare handler with a fresh, unlocked +// mutex in the child, including resetting ThreadSanitizer's mutex metadata. +void resetMutexAfterFork(std::mutex& mutex); +void resetMutexAfterFork(std::recursive_mutex& mutex); + +} // namespace cinderx diff --git a/cinderx/Common/frozen_list.h b/cinderx/Common/frozen_list.h new file mode 100644 index 000000000..69adcb619 --- /dev/null +++ b/cinderx/Common/frozen_list.h @@ -0,0 +1,154 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace cinderx { + +// A frozen list is effectively a vector that is dynamically allocated at +// runtime, but then can no longer be resized. +template + requires std::default_initializable && std::copyable +class FrozenList { + public: + FrozenList() = default; + ~FrozenList() { + if (ptr_) { + std::destroy_n(ptr_, size_); + std::allocator{}.deallocate(ptr_, size_); + } + } + + // Construct a frozen list from the given initializer list. + /* implicit */ FrozenList(std::initializer_list values) { + allocateAndConstruct(values.size(), [&values](T* storage) { + std::uninitialized_copy(values.begin(), values.end(), storage); + }); + } + + FrozenList(const FrozenList& other) { + allocateAndConstruct(other.size_, [&other](T* storage) { + std::uninitialized_copy(other.begin(), other.end(), storage); + }); + } + + FrozenList(FrozenList&& other) noexcept + : ptr_{std::exchange(other.ptr_, nullptr)}, + size_{std::exchange(other.size_, 0)} {} + + FrozenList& operator=(FrozenList&& other) noexcept { + if (this != &other) { + ensureUninitialized(); + + ptr_ = std::exchange(other.ptr_, nullptr); + size_ = std::exchange(other.size_, 0); + } + + return *this; + } + + FrozenList& operator=(const FrozenList& other) { + if (this != &other) { + allocateAndConstruct(other.size_, [&other](T* storage) { + std::uninitialized_copy(other.begin(), other.end(), storage); + }); + } + + return *this; + } + + // The size of the list. + size_t size() const { + return size_; + } + + // Initialize the list with size value-initialized elements. + void initialize(size_t size) { + allocateAndConstruct(size, [size](T* storage) { + std::uninitialized_value_construct_n(storage, size); + }); + } + + // Initialize the list with size copies of the given value. + void initialize(size_t size, const T& val) { + allocateAndConstruct(size, [&val, size](T* storage) { + std::uninitialized_fill_n(storage, size, val); + }); + } + + // Provide the begin function for immutable range-based for-loop support. + const T* begin() const { + return ptr_; + } + + // Provide the end function for immutable range-based for-loop support. + const T* end() const { + return ptr_ + size_; + } + + // Provide the [] operator for accessing elements by index. + T& operator[](size_t index) { + return ptr_[index]; + } + + const T& operator[](size_t index) const { + return ptr_[index]; + } + + // Like the [] operator, but throws an exception if the index is out of range. + T& at(size_t index) { + return const_cast(std::as_const(*this).at(index)); + } + + const T& at(size_t index) const { + if (index >= size_) { + throw std::out_of_range("Index out of range"); + } + return ptr_[index]; + } + + private: + // Raise an exception if the list already owns element storage. + void ensureUninitialized() { + if (ptr_ != nullptr) { + throw std::runtime_error("Cannot initialize FrozenList more than once"); + } + } + + // Construct size elements in newly allocated storage, publishing it only + // after all construction succeeds. constructElements must destroy any + // partially constructed elements before throwing; the uninitialized + // algorithms used by callers provide this guarantee. + template + void allocateAndConstruct(size_t size, ConstructElements constructElements) { + ensureUninitialized(); + + if (size == 0) { + return; + } + + T* storage = std::allocator{}.allocate(size); + + try { + constructElements(storage); + } catch (...) { + // constructElements has already destroyed any partially built elements. + std::allocator{}.deallocate(storage, size); + throw; + } + + ptr_ = storage; + size_ = size; + } + + T* ptr_{nullptr}; + size_t size_{0}; +}; + +} // namespace cinderx diff --git a/cinderx/Common/hugepages.cpp b/cinderx/Common/hugepages.cpp new file mode 100644 index 000000000..766929493 --- /dev/null +++ b/cinderx/Common/hugepages.cpp @@ -0,0 +1,138 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#include "cinderx/Common/hugepages.h" + +#include "cinderx/Common/log.h" +#include "cinderx/Common/util.h" + +#ifndef WIN32 +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef WIN32 +#ifndef MAP_HUGETLB +#define MAP_HUGETLB 0x40000 +#endif +#ifndef MAP_HUGE_SHIFT +#define MAP_HUGE_SHIFT 26 +#endif +#ifndef MAP_HUGE_2MB +#define MAP_HUGE_2MB (21 << MAP_HUGE_SHIFT) +#endif +#endif + +namespace cinderx { + +HugePageArena::HugePageArena(HugePageArena&& other) noexcept { + std::lock_guard lock{other.mutex_}; + fill_ = other.fill_; + end_ = other.end_; + chunks_ = std::move(other.chunks_); + other.fill_ = nullptr; + other.end_ = nullptr; +} + +HugePageArena::~HugePageArena() { + for (const Chunk& chunk : chunks_) { + free_aligned(chunk.ptr); + } +} + +void* HugePageArena::allocate(size_t size, size_t alignment) { + std::lock_guard lock{mutex_}; + + char* candidate = fill_ == nullptr + ? nullptr + : reinterpret_cast( + roundUp(reinterpret_cast(fill_), alignment)); + if (candidate == nullptr || candidate + size > end_) { + // Current chunk is exhausted (or none exists yet). A fresh chunk is aligned + // to at least `alignment` and sized to fit, so the allocation succeeds. + allocateChunk(size, alignment); + candidate = reinterpret_cast( + roundUp(reinterpret_cast(fill_), alignment)); + } + fill_ = candidate + size; + return candidate; +} + +void HugePageArena::allocateChunk(size_t size, size_t alignment) { + size_t chunk_alignment = std::max(alignment, kHugePageSize); + size_t chunk_size = roundUp(size, kHugePageSize); + void* chunk = malloc_aligned(chunk_size, chunk_alignment); + JIT_CHECK(chunk != nullptr, "Failed to allocate {} bytes", chunk_size); +#ifdef MADV_HUGEPAGE + // Advise the kernel to back the chunk with transparent huge pages. + madvise(chunk, chunk_size, MADV_HUGEPAGE); +#endif + chunks_.emplace_back(chunk, chunk_size); + fill_ = static_cast(chunk); + end_ = fill_ + chunk_size; +} + +void HugePageArena::afterForkChild() { +#ifndef WIN32 + void* tmp = nullptr; + size_t tmp_size = 0; + std::lock_guard lock{mutex_}; + for (const Chunk& chunk : chunks_) { + // we can theoretically have chunks that are larger than 2MB but don't + // really + if (tmp == nullptr || tmp_size < chunk.size) { + tmp = realloc(tmp, chunk.size); + tmp_size = chunk.size; + JIT_CHECK(tmp != nullptr, "Failed to allocate {} bytes", chunk.size); + } + + // Fault every page in so the child gets its own private physical pages + // instead of copy-on-write references to the parent. A volatile write + // forces the fault without the compiler optimizing it away. + memcpy(tmp, chunk.ptr, chunk.size); + if (madvise(chunk.ptr, chunk.size, MADV_DONTNEED) != 0) { + JIT_DLOG( + "CINDERX: MADV_DONTNEED failed for {} bytes at {} after fork: {}\n", + chunk.size, + chunk.ptr, + strerror(errno)); + } +#ifdef MADV_HUGEPAGE + if (madvise(chunk.ptr, chunk.size, MADV_HUGEPAGE) != 0) { + JIT_DLOG( + "CINDERX: MADV_HUGEPAGE failed for {} bytes at {} after fork: {}\n", + chunk.size, + chunk.ptr, + strerror(errno)); + } +#endif + memcpy(chunk.ptr, tmp, chunk.size); + } + free(tmp); +#endif +} + +void HugePageArena::atForkPrepare() { + mutex_.lock(); +} + +void HugePageArena::atForkParent() { + mutex_.unlock(); +} + +void HugePageArena::atForkChild() { + // Reuse the storage to get a fresh, unlocked mutex. The inherited one is + // still locked by atForkPrepare() and destroying a locked mutex is + // undefined, so its lifetime is ended without running its destructor. + new (&mutex_) std::mutex{}; +} + +} // namespace cinderx diff --git a/cinderx/Common/hugepages.h b/cinderx/Common/hugepages.h new file mode 100644 index 000000000..84245c4a0 --- /dev/null +++ b/cinderx/Common/hugepages.h @@ -0,0 +1,59 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#pragma once + +#include +#include +#include +#include + +namespace cinderx { + +// Bump allocator that packs sub-allocations into large (2MB) chunks backed by +// transparent huge pages, reducing TLB pressure. Small allocations (e.g. JIT +// slabs) share a chunk instead of each getting its own mapping. Chunks are +// owned by the arena and freed together when it is destroyed. Thread-safe. +class HugePageArena { + public: + // 2MB transparent huge pages. + static constexpr size_t kHugePageSize = 2 * 1024 * 1024; + + HugePageArena() = default; + ~HugePageArena(); + + HugePageArena(const HugePageArena&) = delete; + HugePageArena& operator=(const HugePageArena&) = delete; + + // Each arena owns its own mutex, so the moved-from arena keeps its lock and + // is left empty rather than transferring it. + HugePageArena(HugePageArena&& other) noexcept; + + // Return `size` bytes aligned to at least `alignment`. The memory is owned by + // the arena and must not be freed by the caller. + void* allocate(size_t size, size_t alignment); + + // Re-establish huge page backing for every chunk after a fork(). + void afterForkChild(); + + void atForkPrepare(); + void atForkParent(); + void atForkChild(); + + private: + struct Chunk { + void* ptr; + size_t size; + }; + + void allocateChunk(size_t size, size_t alignment); + + std::mutex mutex_; + char* fill_{nullptr}; + char* end_{nullptr}; + std::vector chunks_; +}; + +// Maintain the pages on huge pages +void hugePagesAfterFork(); + +} // namespace cinderx diff --git a/cinderx/Common/log.cpp b/cinderx/Common/log.cpp index 33b0f0547..b9eeb4939 100644 --- a/cinderx/Common/log.cpp +++ b/cinderx/Common/log.cpp @@ -2,27 +2,101 @@ #include "cinderx/Common/log.h" -#include "cinderx/Jit/threaded_compile.h" +#include "cinderx/Common/py-portability.h" -namespace jit { +#include +#include + +namespace cinderx { + +namespace { + +// Trim file paths to be rooted at "cinderx/" for cleaner log output. +std::string_view trimSourcePath(std::string_view path) { + constexpr std::string_view pattern = +#ifdef _WIN32 + "cinderx\\" +#else + "cinderx/" +#endif + ; + size_t pos = path.rfind(pattern); + return pos != std::string_view::npos ? path.substr(pos) : path; +} + +[[noreturn]] CINDERX_COLD void abortImpl() { + fmt::print(stderr, "\n"); + std::fflush(stderr); + printPythonException(); + std::abort(); +} + +} // namespace + +CINDERX_COLD void logImplV( + std::string_view file, + int line, + fmt::string_view format, + fmt::format_args args) { + FILE* output = jit::getConfig().log.output_file; + static std::mutex mutex; + std::lock_guard lock{mutex}; + fmt::print(output, "JIT: {}:{} -- ", trimSourcePath(file), line); + fmt::vprint(output, format, args); + fmt::print(output, "\n"); + std::fflush(output); +} + +[[noreturn]] CINDERX_COLD void abortImplV( + std::string_view file, + int line, + fmt::string_view format, + fmt::format_args args) { + fmt::print(stderr, "JIT: {}:{} -- Abort\n", trimSourcePath(file), line); + fmt::vprint(stderr, format, args); + abortImpl(); +} + +[[noreturn]] CINDERX_COLD void checkFailedImplV( + std::string_view file, + int line, + std::string_view cond_str, + fmt::string_view format, + fmt::format_args args) { + fmt::print( + stderr, + "JIT: {}:{} -- Assertion failed: {}\n", + trimSourcePath(file), + line, + cond_str); + fmt::vprint(stderr, format, args); + abortImpl(); +} + +[[noreturn]] CINDERX_COLD void throwImplV( + std::string_view file, + int line, + fmt::string_view format, + fmt::format_args args) { + std::string msg = fmt::format("{}:{} ", trimSourcePath(file), line); + fmt::vformat_to(std::back_inserter(msg), format, args); + throw std::runtime_error{msg}; +} void printPythonException() { -#if PY_VERSION_HEX < 0x030C0000 - PyThreadState* tstate = PyThreadState_Get(); - if (tstate != nullptr && tstate->curexc_type != nullptr) { - PyErr_Display( - tstate->curexc_type, tstate->curexc_value, tstate->curexc_traceback); + // This can run on a background compile thread that does not hold the GIL + // (e.g. a JIT_CHECK firing mid-compile). Touching the Python error indicator + // without the GIL is unsafe, so only report when the GIL is held. + if (PyThreadState_GetUnchecked() == nullptr) { + return; } -#else if (PyErr_Occurred()) { - PyErr_DisplayException(PyErr_GetRaisedException()); + auto exc = Ref<>::steal(PyErr_GetRaisedException()); + PyErr_DisplayException(exc); } -#endif } std::string repr(BorrowedRef<> obj) { - jit::ThreadedCompileSerialize guard; - PyObject *t, *v, *tb; PyErr_Fetch(&t, &v, &tb); @@ -41,4 +115,14 @@ std::string repr(BorrowedRef<> obj) { return {str, static_cast(len)}; } -} // namespace jit +void setRuntimeError(const std::exception& exn) { + // Shouldn't happen, but in case we doubled up on Python and C++ exceptions, + // make sure to log the Python exception first, then override it with the C++ + // exception. Otherwise it would just be lost. + if (auto err = Ref<>::steal(PyErr_GetRaisedException())) { + PyErr_DisplayException(err); + } + PyErr_SetString(PyExc_RuntimeError, exn.what()); +} + +} // namespace cinderx diff --git a/cinderx/Common/log.h b/cinderx/Common/log.h index 24c16d034..1bb148630 100644 --- a/cinderx/Common/log.h +++ b/cinderx/Common/log.h @@ -4,9 +4,9 @@ #include "cinderx/python.h" +#include "cinderx/Common/define.h" #include "cinderx/Common/ref.h" #include "cinderx/Jit/config.h" -#include "cinderx/Jit/threaded_compile.h" #include #include @@ -15,8 +15,25 @@ #include #include +#include -namespace jit { +#if defined(__has_cpp_attribute) +#if __has_cpp_attribute(gnu::cold) +#define CINDERX_COLD [[gnu::cold]] +#endif +#endif + +#if !defined(CINDERX_COLD) && defined(__has_attribute) +#if __has_attribute(cold) +#define CINDERX_COLD __attribute__((cold)) +#endif +#endif + +#ifndef CINDERX_COLD +#define CINDERX_COLD +#endif + +namespace cinderx { template auto format_to( @@ -36,78 +53,121 @@ void printPythonException(); // "" std::string repr(BorrowedRef<> obj); -#define JIT_LOG(...) \ - { \ - FILE* _output = jit::getConfig().log.output_file; \ - jit::ThreadedCompileSerialize guard; \ - fmt::print(_output, "JIT: {}:{} -- ", __FILE__, __LINE__); \ - fmt::print(_output, __VA_ARGS__); \ - fmt::print(_output, "\n"); \ - std::fflush(_output); \ - } +// Set a Python RuntimeError from a C++ exception. +// +// Will replace an existing Python exception if one exists, but will log it +// first. +void setRuntimeError(const std::exception& exn); + +// Outlined logging implementations to reduce code size on hot paths. +CINDERX_COLD void logImplV( + std::string_view file, + int line, + fmt::string_view format, + fmt::format_args args); +[[noreturn]] CINDERX_COLD void abortImplV( + std::string_view file, + int line, + fmt::string_view format, + fmt::format_args args); +[[noreturn]] CINDERX_COLD void checkFailedImplV( + std::string_view file, + int line, + std::string_view cond_str, + fmt::string_view format, + fmt::format_args args); +[[noreturn]] CINDERX_COLD void throwImplV( + std::string_view file, + int line, + fmt::string_view format, + fmt::format_args args); + +template +CINDERX_COLD void logImpl( + std::string_view file, + int line, + fmt::format_string format, + Args&&... args) { + logImplV(file, line, format, fmt::make_format_args(args...)); +} + +template +[[noreturn]] CINDERX_COLD void abortImpl( + std::string_view file, + int line, + fmt::format_string format, + Args&&... args) { + abortImplV(file, line, format, fmt::make_format_args(args...)); +} + +template +[[noreturn]] CINDERX_COLD void checkFailedImpl( + std::string_view file, + int line, + std::string_view cond_str, + fmt::format_string format, + Args&&... args) { + checkFailedImplV( + file, line, cond_str, format, fmt::make_format_args(args...)); +} + +template +[[noreturn]] CINDERX_COLD void throwImpl( + std::string_view file, + int line, + fmt::format_string format, + Args&&... args) { + throwImplV(file, line, format, fmt::make_format_args(args...)); +} + +#define JIT_LOG(...) cinderx::logImpl(__FILE__, __LINE__, __VA_ARGS__) #define JIT_LOGIF(PRED, ...) \ if (PRED) { \ JIT_LOG(__VA_ARGS__); \ } -#define JIT_DLOG(...) JIT_LOGIF(jit::getConfig().log.debug, __VA_ARGS__) - -#define JIT_CHECK(COND, ...) \ - { \ - if (!(COND)) { \ - fmt::print( \ - stderr, \ - "JIT: {}:{} -- Assertion failed: {}\n", \ - __FILE__, \ - __LINE__, \ - #COND); \ - JIT_ABORT_IMPL(__VA_ARGS__); \ - } \ +#define JIT_DLOG(...) \ + JIT_LOGIF(cinderx::jit::getConfig().log.debug, __VA_ARGS__) + +#define JIT_CHECK(COND, ...) \ + { \ + if (!(COND)) { \ + cinderx::checkFailedImpl(__FILE__, __LINE__, #COND, __VA_ARGS__); \ + } \ } #define JIT_CHECK_ONCE(COND, ...) \ { \ static bool checked = false; \ if (!checked) { \ - JIT_CHECK(COND, __VA_ARGS__); \ - } else { \ checked = true; \ + JIT_CHECK(COND, __VA_ARGS__); \ } \ } -#define JIT_ABORT(...) \ - { \ - fmt::print(stderr, "JIT: {}:{} -- Abort\n", __FILE__, __LINE__); \ - JIT_ABORT_IMPL(__VA_ARGS__); \ - } +#define JIT_ABORT(...) cinderx::abortImpl(__FILE__, __LINE__, __VA_ARGS__) -#define JIT_ABORT_IMPL(...) \ - { \ - fmt::print(stderr, __VA_ARGS__); \ - fmt::print(stderr, "\n"); \ - std::fflush(stderr); \ - jit::printPythonException(); \ - std::abort(); \ +#define JIT_THROW(...) cinderx::throwImpl(__FILE__, __LINE__, __VA_ARGS__) + +#define JIT_THROW_IF(COND, ...) \ + if (COND) { \ + JIT_THROW(__VA_ARGS__); \ } -#ifdef Py_DEBUG -#define JIT_DABORT(...) JIT_ABORT(__VA_ARGS__) -#define JIT_DCHECK(COND, ...) JIT_CHECK((COND), __VA_ARGS__) -#define JIT_DCHECK_ONCE(COND, ...) JIT_CHECK_ONCE((COND), __VA_ARGS__) -#else #define JIT_DABORT(...) \ - if (0) { \ + if constexpr (kDebug) { \ JIT_ABORT(__VA_ARGS__); \ } + #define JIT_DCHECK(COND, ...) \ - if (0) { \ + if constexpr (kDebug) { \ JIT_CHECK((COND), __VA_ARGS__); \ } + #define JIT_DCHECK_ONCE(COND, ...) \ - if (0) { \ + if constexpr (kDebug) { \ JIT_CHECK_ONCE((COND), __VA_ARGS__); \ } -#endif -} // namespace jit +} // namespace cinderx diff --git a/cinderx/Common/long.cpp b/cinderx/Common/long.cpp new file mode 100644 index 000000000..6febe7762 --- /dev/null +++ b/cinderx/Common/long.cpp @@ -0,0 +1,25 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#include "cinderx/Common/long.h" + +#include "cinderx/Common/log.h" + +extern "C" { + +#include "internal/pycore_long.h" + +} // extern "C" + +namespace cinderx { + +BorrowedRef smallInt(int32_t n) { + JIT_THROW_IF( + n < -_PY_NSMALLNEGINTS || n >= _PY_NSMALLPOSINTS, + "{} is out of bounds for small Python longs ([{}, {}])", + n, + -_PY_NSMALLNEGINTS, + _PY_NSMALLPOSINTS - 1); + return &_PyLong_SMALL_INTS[n + _PY_NSMALLNEGINTS]; +} + +} // namespace cinderx diff --git a/cinderx/Common/long.h b/cinderx/Common/long.h new file mode 100644 index 000000000..fed566400 --- /dev/null +++ b/cinderx/Common/long.h @@ -0,0 +1,16 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#pragma once + +#include "cinderx/python.h" + +#include "cinderx/Common/ref.h" + +#include + +namespace cinderx { + +// Load the PyLongObject for a small integer (from -5 to 256, inclusive). +BorrowedRef smallInt(int32_t n); + +} // namespace cinderx diff --git a/cinderx/Common/opcode_stubs.h b/cinderx/Common/opcode_stubs.h index a007d248b..b618dcf55 100644 --- a/cinderx/Common/opcode_stubs.h +++ b/cinderx/Common/opcode_stubs.h @@ -14,123 +14,7 @@ // Having them defined across all builds means there can be less Python version // checks in the compiler. -#if PY_VERSION_HEX < 0x030C0000 - -#define STUB_OPCODE_DEFS(X) \ - X(BEFORE_WITH) \ - X(BINARY_OP) \ - X(BINARY_OP_ADD_INT) \ - X(BINARY_OP_MULTIPLY_INT) \ - X(BINARY_OP_SUBTRACT_INT) \ - X(BINARY_OP_ADD_FLOAT) \ - X(BINARY_OP_MULTIPLY_FLOAT) \ - X(BINARY_OP_SUBTRACT_FLOAT) \ - X(BINARY_OP_ADD_UNICODE) \ - X(BINARY_SLICE) \ - X(BINARY_SUBSCR_LIST_INT) \ - X(BINARY_SUBSCR_TUPLE_INT) \ - X(BUILD_INTERPOLATION) \ - X(BUILD_TEMPLATE) \ - X(CACHE) \ - X(CALL) \ - X(CALL_INTRINSIC_1) \ - X(CALL_INTRINSIC_2) \ - X(CALL_KW) \ - X(CHECK_EG_MATCH) \ - X(CHECK_EXC_MATCH) \ - X(CLEANUP_THROW) \ - X(COMPARE_OP_FLOAT) \ - X(COMPARE_OP_INT) \ - X(COMPARE_OP_STR) \ - X(CONVERT_VALUE) \ - X(COPY) \ - X(COPY_FREE_VARS) \ - X(EAGER_IMPORT_NAME) \ - X(END_FOR) \ - X(END_SEND) \ - X(EXTENDED_OPCODE) \ - X(FORMAT_WITH_SPEC) \ - X(FORMAT_SIMPLE) \ - X(INSTRUMENTED_CALL) \ - X(INSTRUMENTED_CALL_FUNCTION_EX) \ - X(INSTRUMENTED_END_FOR) \ - X(INSTRUMENTED_END_SEND) \ - X(INSTRUMENTED_FOR_ITER) \ - X(INSTRUMENTED_INSTRUCTION) \ - X(INSTRUMENTED_JUMP_BACKWARD) \ - X(INSTRUMENTED_JUMP_FORWARD) \ - X(INSTRUMENTED_LINE) \ - X(INSTRUMENTED_LOAD_SUPER_ATTR) \ - X(INSTRUMENTED_POP_JUMP_IF_FALSE) \ - X(INSTRUMENTED_POP_JUMP_IF_NONE) \ - X(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) \ - X(INSTRUMENTED_POP_JUMP_IF_TRUE) \ - X(INSTRUMENTED_RESUME) \ - X(INSTRUMENTED_RETURN_CONST) \ - X(INSTRUMENTED_RETURN_VALUE) \ - X(INSTRUMENTED_YIELD_VALUE) \ - X(JUMP_BACKWARD) \ - X(JUMP_BACKWARD_NO_INTERRUPT) \ - X(KW_NAMES) \ - X(LOAD_COMMON_CONSTANT) \ - X(LOAD_FAST_AND_CLEAR) \ - X(LOAD_FAST_CHECK) \ - X(LOAD_FAST_BORROW) \ - X(LOAD_FAST_BORROW_LOAD_FAST_BORROW) \ - X(LOAD_FAST_LOAD_FAST) \ - X(LOAD_FROM_DICT_OR_DEREF) \ - X(LOAD_FROM_DICT_OR_GLOBALS) \ - X(LOAD_LOCALS) \ - X(LOAD_SMALL_INT) \ - X(LOAD_SPECIAL) \ - X(LOAD_SUPER_ATTR) \ - X(MAKE_CELL) \ - X(NOT_TAKEN) \ - X(POP_ITER) \ - X(POP_JUMP_IF_NONE) \ - X(POP_JUMP_IF_NOT_NONE) \ - X(PUSH_EXC_INFO) \ - X(PUSH_NULL) \ - X(RESUME) \ - X(RETURN_CONST) \ - X(RETURN_GENERATOR) \ - X(SEND) \ - X(SET_FUNCTION_ATTRIBUTE) \ - X(STORE_FAST_STORE_FAST) \ - X(STORE_FAST_LOAD_FAST) \ - X(STORE_SLICE) \ - X(STORE_SUBSCR_DICT) \ - X(SWAP) \ - X(TO_BOOL) \ - X(UNPACK_SEQUENCE_LIST) \ - X(UNPACK_SEQUENCE_TUPLE) \ - X(UNPACK_SEQUENCE_TWO_TUPLE) - -#define STUB_NB_DEFS(X) \ - X(ADD) \ - X(AND) \ - X(FLOOR_DIVIDE) \ - X(LSHIFT) \ - X(MATRIX_MULTIPLY) \ - X(MULTIPLY) \ - X(REMAINDER) \ - X(OR) \ - X(POWER) \ - X(RSHIFT) \ - X(SUBTRACT) \ - X(TRUE_DIVIDE) \ - X(XOR) - -enum { -#define DEFINE_NB(X) NB_##X, - STUB_NB_DEFS(DEFINE_NB) -#undef DEFINE_NB -#define DEFINE_NB_INPLACE(X) NB_INPLACE_##X, - STUB_NB_DEFS(DEFINE_NB_INPLACE) -#undef DEFINE_NB_INPLACE -}; - -#elif PY_VERSION_HEX < 0x030E0000 +#if PY_VERSION_HEX < 0x030E0000 #define STUB_OPCODE_DEFS(X) \ X(BINARY_ADD) \ @@ -140,6 +24,9 @@ enum { X(BINARY_MATRIX_MULTIPLY) \ X(BINARY_MODULO) \ X(BINARY_MULTIPLY) \ + X(BINARY_OP_SUBSCR_DICT) \ + X(BINARY_OP_SUBSCR_LIST_INT) \ + X(BINARY_OP_SUBSCR_TUPLE_INT) \ X(BINARY_OR) \ X(BINARY_POWER) \ X(BINARY_RSHIFT) \ @@ -227,9 +114,109 @@ enum { X(STORE_FAST_STORE_FAST) \ X(STORE_FAST_LOAD_FAST) \ X(TO_BOOL) \ + X(TO_BOOL_BOOL) \ + X(TO_BOOL_INT) \ + X(TO_BOOL_LIST) \ + X(TO_BOOL_NONE) \ + X(TO_BOOL_STR) \ X(UNARY_POSITIVE) \ X(YIELD_FROM) +#elif PY_VERSION_HEX < 0x030F0000 + +#define STUB_OPCODE_DEFS(X) \ + X(BEFORE_ASYNC_WITH) \ + X(BEFORE_WITH) \ + X(BINARY_ADD) \ + X(BINARY_AND) \ + X(BINARY_FLOOR_DIVIDE) \ + X(BINARY_LSHIFT) \ + X(BINARY_MATRIX_MULTIPLY) \ + X(BINARY_MODULO) \ + X(BINARY_MULTIPLY) \ + X(BINARY_OR) \ + X(BINARY_POWER) \ + X(BINARY_RSHIFT) \ + X(BINARY_SUBSCR) \ + X(BINARY_SUBSCR_DICT_STR) \ + X(BINARY_SUBSCR_LIST) \ + X(BINARY_SUBSCR_TUPLE) \ + X(BINARY_SUBSCR_TUPLE_CONST_INT) \ + X(BINARY_SUBSCR_TUPLE_INT) \ + X(BINARY_SUBSCR_LIST_INT) \ + X(BINARY_SUBSCR_DICT) \ + X(BINARY_SUBTRACT) \ + X(BINARY_TRUE_DIVIDE) \ + X(BINARY_XOR) \ + X(BUILD_CONST_KEY_MAP) \ + X(CALL_FUNCTION) \ + X(CALL_FUNCTION_KW) \ + X(CALL_METHOD) \ + X(COPY_DICT_WITHOUT_KEYS) \ + X(DUP_TOP) \ + X(DUP_TOP_TWO) \ + X(FORMAT_VALUE) \ + X(GEN_START) \ + X(INPLACE_ADD) \ + X(INPLACE_AND) \ + X(INPLACE_FLOOR_DIVIDE) \ + X(INPLACE_LSHIFT) \ + X(INPLACE_MATRIX_MULTIPLY) \ + X(INPLACE_MODULO) \ + X(INPLACE_MULTIPLY) \ + X(INPLACE_OR) \ + X(INPLACE_POWER) \ + X(INPLACE_RSHIFT) \ + X(INPLACE_SUBTRACT) \ + X(INPLACE_TRUE_DIVIDE) \ + X(INPLACE_XOR) \ + X(JUMP_ABSOLUTE) \ + X(JUMP_IF_FALSE_OR_POP) \ + X(JUMP_IF_NOT_EXC_MATCH) \ + X(JUMP_IF_NONZERO_OR_POP) \ + X(JUMP_IF_TRUE_OR_POP) \ + X(JUMP_IF_ZERO_OR_POP) \ + X(KW_NAMES) \ + X(LIST_TO_TUPLE) \ + X(LOAD_ASSERTION_ERROR) \ + X(LOAD_ATTR_DICT_DESCR) \ + X(LOAD_ATTR_DICT_NO_DESCR) \ + X(LOAD_ATTR_NO_DICT_DESCR) \ + X(LOAD_ATTR_POLYMORPHIC) \ + X(LOAD_ATTR_SPLIT_DICT) \ + X(LOAD_ATTR_SPLIT_DICT_DESCR) \ + X(LOAD_ATTR_SUPER) \ + X(LOAD_ATTR_S_MODULE) \ + X(LOAD_ATTR_TYPE) \ + X(LOAD_ATTR_UNCACHABLE) \ + X(LOAD_METHOD) \ + X(LOAD_METHOD_DICT_DESCR) \ + X(LOAD_METHOD_DICT_METHOD) \ + X(LOAD_METHOD_MODULE) \ + X(LOAD_METHOD_NO_DICT_DESCR) \ + X(LOAD_METHOD_NO_DICT_METHOD) \ + X(LOAD_METHOD_SPLIT_DICT_DESCR) \ + X(LOAD_METHOD_SPLIT_DICT_METHOD) \ + X(LOAD_METHOD_SUPER) \ + X(LOAD_METHOD_S_MODULE) \ + X(LOAD_METHOD_TYPE) \ + X(LOAD_METHOD_TYPE_METHODLIKE) \ + X(LOAD_METHOD_UNCACHABLE) \ + X(LOAD_METHOD_UNSHADOWED_METHOD) \ + X(MAKE_OPNAME) \ + X(RETURN_CONST) \ + X(ROT_FOUR) \ + X(ROT_N) \ + X(ROT_THREE) \ + X(ROT_TWO) \ + X(SETUP_ASYNC_WITH) \ + X(STORE_ATTR_DESCR) \ + X(STORE_ATTR_DICT) \ + X(STORE_ATTR_SPLIT_DICT) \ + X(STORE_ATTR_UNCACHABLE) \ + X(UNARY_POSITIVE) \ + X(YIELD_FROM) + #else #define STUB_OPCODE_DEFS(X) \ @@ -265,6 +252,7 @@ enum { X(DUP_TOP_TWO) \ X(FORMAT_VALUE) \ X(GEN_START) \ + X(GET_YIELD_FROM_ITER) \ X(INPLACE_ADD) \ X(INPLACE_AND) \ X(INPLACE_FLOOR_DIVIDE) \ diff --git a/cinderx/Common/py-portability.h b/cinderx/Common/py-portability.h index 56161b39d..a95456dcd 100644 --- a/cinderx/Common/py-portability.h +++ b/cinderx/Common/py-portability.h @@ -15,19 +15,10 @@ #include "internal/pycore_genobject.h" #endif -#if PY_VERSION_HEX < 0x030C0000 -#define CI_INTERP_IMPORT_FIELD(interp, field) interp->field -#else #define CI_INTERP_IMPORT_FIELD(interp, field) interp->imports.field -#endif #include "internal/pycore_interp.h" -#if PY_VERSION_HEX < 0x030C0000 -#define _PyType_GetDict(type) ((type)->tp_dict) -#define _PyObject_CallNoArgs _PyObject_CallNoArg -#endif - #if PY_VERSION_HEX < 0x030D0000 // Basic renames that went into 3.13. @@ -63,10 +54,13 @@ static inline int PyTime_MonotonicRaw(PyTime_t* result) { // Basic renames that went into 3.14. #define _PyGen_GetGeneratorFromFrame _PyFrame_GetGenerator +#define PyThreadState_GetUnchecked _PyThreadState_UncheckedGet -#endif +// Technically the internal version is different in that it doesn't check types, +// but we always use it with unicode objects. +#define PyUnicode_Equal _PyUnicode_Equal -#if PY_VERSION_HEX >= 0x030C0000 +#endif // Fetch a _PyInterpreterFrame from a PyThreadState. inline _PyInterpreterFrame* interpFrameFromThreadState(PyThreadState* tstate) { @@ -136,8 +130,6 @@ inline void setFrameInstruction(_PyInterpreterFrame* frame, _Py_CODEUNIT* loc) { #endif } -#endif // PY_VERSION_HEX >= 0x030C0000 - #if PY_VERSION_HEX >= 0x030E0000 #define _CiArg_UnpackKeywords( \ args, nargs, kwargs, kwnames, parser, minpos, maxpos, minkw, buf) \ @@ -175,7 +167,7 @@ static inline PyCodeObject* frameCode(_PyInterpreterFrame* frame) { static inline void setFrameCode(_PyInterpreterFrame* frame, PyObject* code) { frame->f_executable = PyStackRef_FromPyObjectNew(code); } -#elif PY_VERSION_HEX >= 0x30C0000 +#else #define FRAME_EXECUTABLE f_code #define FRAME_EXECUTABLE_OFFSET offsetof(_PyInterpreterFrame, f_code) #define FRAME_INSTR prev_instr @@ -187,13 +179,8 @@ inline PyCodeObject* frameCode(_PyInterpreterFrame* frame) { static inline void setFrameCode(_PyInterpreterFrame* frame, PyObject* code) { frame->f_code = (PyCodeObject*)Py_NewRef(code); } -#else -inline PyCodeObject* frameCode(PyFrameObject* frame) { - return frame->f_code; -} #endif -#if PY_VERSION_HEX >= 0x030C0000 static inline PyObject* frameExecutable(_PyInterpreterFrame* frame) { #if PY_VERSION_HEX >= 0x030E0000 return PyStackRef_AsPyObjectBorrow(frame->f_executable); @@ -201,7 +188,6 @@ static inline PyObject* frameExecutable(_PyInterpreterFrame* frame) { return (PyObject*)frameCode(frame); #endif } -#endif // Code object flag that will prevent JIT compilation. // diff --git a/cinderx/Common/ref.cpp b/cinderx/Common/ref.cpp index 7a31d6874..057065bb7 100644 --- a/cinderx/Common/ref.cpp +++ b/cinderx/Common/ref.cpp @@ -6,9 +6,7 @@ #include "internal/pycore_interp.h" #endif -#if PY_VERSION_HEX >= 0x030C0000 #include "internal/pycore_pystate.h" -#endif #if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) #include "internal/pycore_tstate.h" @@ -16,44 +14,30 @@ #include #endif -#ifdef Py_GIL_DISABLED - -void incref_total(PyThreadState* tstate) { -#ifdef Py_REF_DEBUG +void incref_total([[maybe_unused]] PyThreadState* tstate) { +#if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) _PyThreadStateImpl* tstate_impl = (_PyThreadStateImpl*)tstate; std::atomic_ref(tstate_impl->reftotal) .fetch_add(1, std::memory_order_relaxed); #endif } -void decref_total(PyThreadState* tstate) { -#ifdef Py_REF_DEBUG +void decref_total([[maybe_unused]] PyThreadState* tstate) { +#if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) _PyThreadStateImpl* tstate_impl = (_PyThreadStateImpl*)tstate; std::atomic_ref(tstate_impl->reftotal) .fetch_sub(1, std::memory_order_relaxed); #endif } -#else - -void incref_total(PyInterpreterState* interp) { -#ifdef Py_REF_DEBUG -#if PY_VERSION_HEX >= 0x030C0000 +void incref_total([[maybe_unused]] PyInterpreterState* interp) { +#if defined(Py_REF_DEBUG) && !defined(Py_GIL_DISABLED) interp->object_state.reftotal++; -#else - _Py_RefTotal++; -#endif #endif } -void decref_total(PyInterpreterState* interp) { -#ifdef Py_REF_DEBUG -#if PY_VERSION_HEX >= 0x030C0000 +void decref_total([[maybe_unused]] PyInterpreterState* interp) { +#if defined(Py_REF_DEBUG) && !defined(Py_GIL_DISABLED) interp->object_state.reftotal--; -#else - _Py_RefTotal--; -#endif #endif } - -#endif diff --git a/cinderx/Common/ref.h b/cinderx/Common/ref.h index 9c5ad9ea6..e5b4d9789 100644 --- a/cinderx/Common/ref.h +++ b/cinderx/Common/ref.h @@ -27,7 +27,7 @@ class RefBase { return getObj(); } - T* release() { + [[nodiscard]] T* release() { auto ref = ptr_; ptr_ = nullptr; return ref; @@ -65,13 +65,10 @@ class RefBase { T* ptr_{nullptr}; }; -#if defined(Py_GIL_DISABLED) void incref_total(PyThreadState* tstate); void decref_total(PyThreadState* tstate); -#else void incref_total(PyInterpreterState* interp); void decref_total(PyInterpreterState* interp); -#endif /* * BorrowedRef owns a borrowed reference to a PyObject. @@ -129,7 +126,7 @@ class BorrowedRef : public RefBase { template struct std::hash> { - size_t operator()(const BorrowedRef& ref) const { + size_t operator()(const BorrowedRef& ref) const noexcept { std::hash hasher; return hasher(ref.get()); } @@ -288,7 +285,7 @@ struct std::hash> { }; template -struct TransparentRefHasher { +struct RefHasher { using is_transparent = void; size_t operator()(const BorrowedRef& ref) const { @@ -299,3 +296,25 @@ struct TransparentRefHasher { return std::hash>{}(ref); } }; + +// Comparison implementation for Ref and BorrowedRef. +template +struct RefLess { + using is_transparent = void; + + bool operator()(const RefBase& lhs, const RefBase& rhs) const { + return std::less{}(lhs.get(), rhs.get()); + } + + template + requires(!std::same_as && (IsPyObject || IsPyObject)) + bool operator()(const RefBase& lhs, const RefBase& rhs) const { + return std::less{}(lhs.getObj(), rhs.getObj()); + } + + template + requires(!std::same_as && (IsPyObject || IsPyObject)) + bool operator()(const RefBase& lhs, const RefBase& rhs) const { + return std::less{}(lhs.getObj(), rhs.getObj()); + } +}; diff --git a/cinderx/Common/slab.h b/cinderx/Common/slab.h index a353e7230..f8eb5dc1a 100644 --- a/cinderx/Common/slab.h +++ b/cinderx/Common/slab.h @@ -2,6 +2,8 @@ #pragma once +#include "cinderx/Common/aligned_memory.h" +#include "cinderx/Common/hugepages.h" #include "cinderx/Common/log.h" #include "cinderx/Common/util.h" #ifndef WIN32 @@ -9,10 +11,11 @@ #endif #include -#include +#include +#include #include -namespace jit { +namespace cinderx { template class SlabIterator { @@ -44,10 +47,6 @@ class SlabIterator { return ptr_ == o.ptr_; } - bool operator!=(const SlabIterator& o) const { - return !operator==(o); - } - private: char* ptr_{nullptr}; size_t increment_{0}; @@ -61,27 +60,33 @@ class Slab { public: using iterator = SlabIterator; - explicit Slab(size_t increment) : increment_{increment} { + explicit Slab( + size_t increment, + std::shared_ptr arena = nullptr) + : arena_(arena), increment_{increment} { JIT_CHECK( increment >= sizeof(T), "Trying to fit a slab object into too little memory"); - void* ptr; -#ifndef WIN32 - int result = posix_memalign(&ptr, kPageSize, kSlabSize); - JIT_CHECK(result == 0, "Failed to allocate {} bytes", kSlabSize); -#else - ptr = _aligned_malloc(kSlabSize, kPageSize); - JIT_CHECK(ptr != nullptr, "Failed to allocate {} bytes", kSlabSize); -#endif - base_.reset(static_cast(ptr)); - fill_ = base_.get(); + void* ptr = nullptr; + if (arena_ != nullptr) { + ptr = arena_->allocate(kSlabSize, kPageSize); + } + + if (ptr == nullptr) { + owned_base_.emplace(kSlabSize, kPageSize); + ptr = owned_base_->get(); + } + base_ = fill_ = static_cast(ptr); } - Slab(Slab&& other) - : base_{std::move(other.base_)}, + Slab(Slab&& other) noexcept + : base_{other.base_}, + arena_(std::move(other.arena_)), + owned_base_{std::move(other.owned_base_)}, fill_{other.fill_}, increment_{other.increment_} { other.fill_ = nullptr; + other.base_ = nullptr; } ~Slab() { @@ -90,23 +95,25 @@ class Slab { } } - // Allocate memory for a new T object. Returns void* because the object is not - // constructed yet. - void* allocate() { - char* new_fill = fill_ + increment_; - if (new_fill > base_.get() + kSlabSize) { + // Construct a T in the next slot. Returns nullptr when the slab is full and + // leaves the slab unchanged if construction throws. + template + T* emplace(Args&&... args) { + const size_t used = static_cast(fill_ - base_); + if (increment_ > kSlabSize - used) { return nullptr; } - char* ptr = fill_; - fill_ = new_fill; - return ptr; + T* object = std::construct_at( + reinterpret_cast(fill_), std::forward(args)...); + fill_ += increment_; + return object; } #ifndef WIN32 void mlock() { - if (::mlock(base_.get(), kSlabSize) < 0) { - JIT_LOG("Failed to mlock slab at {}", base_.get()); + if (::mlock(base_, kSlabSize) < 0) { + JIT_LOG("Failed to mlock slab at {}", base_); return; } mlocks_++; @@ -119,8 +126,8 @@ class Slab { JIT_LOG("Trying to unlock slab more than it has been been locked"); } - if (::munlock(base_.get(), kSlabSize) < 0) { - JIT_LOG("Failed to munlock slab at {}", base_.get()); + if (::munlock(base_, kSlabSize) < 0) { + JIT_LOG("Failed to munlock slab at {}", base_); return; } mlocks_--; @@ -128,7 +135,7 @@ class Slab { #endif iterator begin() const { - return iterator{base_.get(), increment_}; + return iterator{base_, increment_}; } iterator end() const { @@ -136,10 +143,12 @@ class Slab { } private: - unique_c_ptr base_; + char* base_; + std::shared_ptr arena_; + std::optional> owned_base_; char* fill_{nullptr}; size_t increment_{0}; size_t mlocks_{0}; }; -} // namespace jit +} // namespace cinderx diff --git a/cinderx/Common/slab_arena.cpp b/cinderx/Common/slab_arena.cpp new file mode 100644 index 000000000..db1fc71ff --- /dev/null +++ b/cinderx/Common/slab_arena.cpp @@ -0,0 +1,75 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#include "cinderx/Common/slab_arena.h" + +#include "cinderx/Common/fork_support.h" +#include "cinderx/module_state.h" + +#include + +#if defined(__linux__) && defined(__aarch64__) +// On ARM64 we see huge dTLB misses on our inline caches so +// we put them on huge pages +#define ALLOCATE_HUGE_PAGES +#endif + +namespace cinderx { + +std::shared_ptr getSharedHugePageArena() { +#ifdef ALLOCATE_HUGE_PAGES + auto state = getModuleState(); + if (state != nullptr) { + return state->getSharedHugePageArena(); + } +#endif + return nullptr; +} + +SlabArenaForkRegistry& SlabArenaForkRegistry::get() { + // Deliberately leaked: SlabArenas are owned by the CinderX module state, + // which can outlive static destructors and would then unregister into a + // destroyed object. + static auto* registry = new SlabArenaForkRegistry; + return *registry; +} + +void SlabArenaForkRegistry::add(std::mutex* mutex) { + std::lock_guard guard{lock_}; + mutexes_.push_back(mutex); +} + +void SlabArenaForkRegistry::remove(std::mutex* mutex) { + auto it = std::find(mutexes_.begin(), mutexes_.end(), mutex); + if (it != mutexes_.end()) { + *it = std::move(mutexes_.back()); + mutexes_.pop_back(); + } +} + +void SlabArenaForkRegistry::atForkPrepare() { + // Holding lock_ across the fork also stops the list itself from being + // mutated while it's being walked. + lock_.lock(); + for (std::mutex* mutex : mutexes_) { + mutex->lock(); + } +} + +void SlabArenaForkRegistry::atForkParent() { + for (std::mutex* mutex : mutexes_) { + mutex->unlock(); + } + lock_.unlock(); +} + +void SlabArenaForkRegistry::atForkChild() { + // Reuse each mutex's storage to get a fresh, unlocked one. They're all + // still locked by atForkPrepare() and destroying a locked mutex is + // undefined, so their lifetimes end without running their destructors. + for (std::mutex* mutex : mutexes_) { + resetMutexAfterFork(*mutex); + } + resetMutexAfterFork(lock_); +} + +} // namespace cinderx diff --git a/cinderx/Common/slab_arena.h b/cinderx/Common/slab_arena.h index e5810a7a8..03fefe41d 100644 --- a/cinderx/Common/slab_arena.h +++ b/cinderx/Common/slab_arena.h @@ -2,17 +2,19 @@ #pragma once +#include "cinderx/Common/hugepages.h" #include "cinderx/Common/log.h" #include "cinderx/Common/slab.h" #include "cinderx/Common/util.h" #include #include +#include #include #include #include -namespace jit { +namespace cinderx { template struct ObjectSizeTrait { @@ -40,7 +42,6 @@ class SlabArenaIterator { } bool operator==(const SlabArenaIterator& other) const = default; - bool operator!=(const SlabArenaIterator& other) const = default; T& operator*() { return *slab_iter_; @@ -58,7 +59,11 @@ class SlabArenaIterator { return *this = SlabArenaIterator{}; } slab_iter_ = currentSlab().begin(); - JIT_CHECK(slab_iter_ != currentSlab().end(), "Unexpected empty slab"); + // Only the last slab can be empty: a new slab is appended only when the + // previous one is full, and an empty slab accepts the next allocation. + if (isSlabEnd()) { + return *this = SlabArenaIterator{}; + } } return *this; } @@ -90,6 +95,30 @@ class SlabArenaIterator { SlabIterator slab_iter_; }; +std::shared_ptr getSharedHugePageArena(); + +// The mutexes of every live SlabArena, so that pthread_atfork() handlers can +// quiesce them across a fork(). +// +// SlabArena is a template with instances scattered across JIT state, so they +// register themselves here instead of being enumerated by hand. No SlabArena +// ever locks another, so the handlers may take them in any order. +class SlabArenaForkRegistry { + public: + static SlabArenaForkRegistry& get(); + + void add(std::mutex* mutex); + void remove(std::mutex* mutex); + + void atForkPrepare(); + void atForkParent(); + void atForkChild(); + + private: + std::mutex lock_; + std::vector mutexes_; +}; + // SlabArena is a simple arena allocator, using slabs that are multiples of the // system's page size. Allocated objects never move after creation, and all // objects will be kept alive until the SlabArena they came from is destroyed. @@ -115,9 +144,21 @@ class SlabArena { using iterator = SlabArenaIterator; SlabArena() { - slabs_.emplace_back(SizeTrait::size()); + slabs_.emplace_back(SizeTrait::size(), getSharedHugePageArena()); + // Registered last so a throwing constructor can't leave a dangling pointer + // behind, as the destructor won't run for a half-constructed arena. + SlabArenaForkRegistry::get().add(&mutex_); } + ~SlabArena() { + SlabArenaForkRegistry::get().remove(&mutex_); + } + + SlabArena(const SlabArena&) = delete; + SlabArena(SlabArena&&) = delete; + SlabArena& operator=(const SlabArena&) = delete; + SlabArena& operator=(SlabArena&&) = delete; + // Allocate a new instance of T using the given constructor arguments. template T* allocate(Args&&... args) { @@ -131,17 +172,19 @@ class SlabArena { } #endif - void* mem = slabs_.back().allocate(); - if (mem == nullptr) { - mem = slabs_.emplace_back(SizeTrait::size()).allocate(); - JIT_CHECK(mem != nullptr, "Empty slab failed to allocate"); + T* object = slabs_.back().emplace(std::forward(args)...); + if (object == nullptr) { + auto& slab = + slabs_.emplace_back(SizeTrait::size(), getSharedHugePageArena()); #ifndef WIN32 if (mlocked_) { - slabs_.back().mlock(); + slab.mlock(); } #endif + object = slab.emplace(std::forward(args)...); + JIT_CHECK(object != nullptr, "Empty slab failed to allocate"); } - return new (mem) T(std::forward(args)...); + return object; } #ifndef WIN32 @@ -178,4 +221,4 @@ class SlabArena { #endif }; -} // namespace jit +} // namespace cinderx diff --git a/cinderx/Common/sorted_vec_map.h b/cinderx/Common/sorted_vec_map.h new file mode 100644 index 000000000..1221ddb68 --- /dev/null +++ b/cinderx/Common/sorted_vec_map.h @@ -0,0 +1,129 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#pragma once + +#include +#include +#include +#include +#include + +namespace cinderx { + +// An associative map backed by a single sorted std::vector of key/value pairs. +// +// This is aimed at small maps that are populated once and then read: lookups +// are O(log n) binary searches, iteration visits keys in ascending order with +// good cache locality, and the whole thing is one contiguous allocation. +// Insertion is O(n) because it shifts later elements, which is fine for the +// small sizes we use it for. +template > +class SortedVecMap { + template + static constexpr bool kIsTransparentLookup = + requires(const Compare& comp, const Key& key, const K& lookup) { + typename Compare::is_transparent; + { comp(key, lookup) } -> std::convertible_to; + { comp(lookup, key) } -> std::convertible_to; + }; + + public: + using key_type = Key; + using mapped_type = Value; + using value_type = std::pair; + using container_type = std::vector; + using iterator = typename container_type::iterator; + using const_iterator = typename container_type::const_iterator; + using size_type = typename container_type::size_type; + + iterator begin() { + return data_.begin(); + } + + iterator end() { + return data_.end(); + } + + const_iterator begin() const { + return data_.begin(); + } + + const_iterator end() const { + return data_.end(); + } + + bool empty() const { + return data_.empty(); + } + + size_type size() const { + return data_.size(); + } + + iterator find(const Key& key) { + iterator it = lowerBound(key); + return matches(it, key) ? it : data_.end(); + } + + const_iterator find(const Key& key) const { + const_iterator it = lowerBound(key); + return matches(it, key) ? it : data_.end(); + } + + template + requires kIsTransparentLookup + iterator find(const K& key) { + iterator it = lowerBound(key); + return matches(it, key) ? it : data_.end(); + } + + template + requires kIsTransparentLookup + const_iterator find(const K& key) const { + const_iterator it = lowerBound(key); + return matches(it, key) ? it : data_.end(); + } + + // Insert a key/value pair, keeping the backing vector sorted by key. If the + // key is already present nothing is inserted. Return an iterator to the + // element with that key and whether a new element was inserted. + template + std::pair emplace(K&& key, V&& value) { + iterator it = lowerBound(key); + if (matches(it, key)) { + return {it, false}; + } + iterator inserted = + data_.emplace(it, std::forward(key), std::forward(value)); + return {inserted, true}; + } + + private: + template + bool matches(const_iterator it, const K& key) const { + return it != data_.end() && !comp_(key, it->first); + } + + template + iterator lowerBound(const K& key) { + return std::lower_bound( + data_.begin(), + data_.end(), + key, + [this](const value_type& a, const K& b) { return comp_(a.first, b); }); + } + + template + const_iterator lowerBound(const K& key) const { + return std::lower_bound( + data_.begin(), + data_.end(), + key, + [this](const value_type& a, const K& b) { return comp_(a.first, b); }); + } + + container_type data_; + [[no_unique_address]] Compare comp_; +}; + +} // namespace cinderx diff --git a/cinderx/Common/string.cpp b/cinderx/Common/string.cpp new file mode 100644 index 000000000..aa3e23a34 --- /dev/null +++ b/cinderx/Common/string.cpp @@ -0,0 +1,26 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#include "cinderx/Common/string.h" + +#include "cinderx/Common/log.h" +#include "cinderx/Common/py-portability.h" + +extern "C" PyObject* Ci_InitStaticStringImpl(const char* s) { + PyObject* obj = PyUnicode_FromString(s); + JIT_CHECK( + obj != nullptr, + "Fatal error, failed to initialize static string '{}'", + s); + +#if PY_VERSION_HEX >= 0x030E0000 && defined(Py_GIL_DISABLED) + obj->ob_tid = _Py_UNOWNED_TID; + obj->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; + obj->ob_ref_shared = 0; + _Py_atomic_or_uint8(&obj->ob_gc_bits, _PyGC_BITS_DEFERRED); + _PyASCIIObject_CAST(obj)->state.statically_allocated = 0; +#else + obj->ob_refcnt = 0x3fffffff; +#endif + + return obj; +} diff --git a/cinderx/Common/string.h b/cinderx/Common/string.h index d68471c5d..0185f9f01 100644 --- a/cinderx/Common/string.h +++ b/cinderx/Common/string.h @@ -4,34 +4,27 @@ #include "cinderx/python.h" -#include "cinderx/Common/py-portability.h" +#ifdef __cplusplus +extern "C" { +#endif // Create a function static variable for Python string. This string is // explicitly immortalized, but not interned because doing so will cause it to // be released by the runtime at shutdown. We cannot use // _Py_SetImmortalUntracked() as this has an assertion to force the use of // _PyUnicode_InternImmortal() for strings. -#if PY_VERSION_HEX >= 0x030E0000 && defined(Py_GIL_DISABLED) -#define DEFINE_NAMED_STATIC_STRING(NAME, STR) \ - static PyObject* NAME = NULL; \ - if (NAME == NULL) { \ - PyObject* op = PyUnicode_FromString(STR); \ - op->ob_tid = _Py_UNOWNED_TID; \ - op->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; \ - op->ob_ref_shared = 0; \ - _Py_atomic_or_uint8(&op->ob_gc_bits, _PyGC_BITS_DEFERRED); \ - _PyASCIIObject_CAST(op)->state.statically_allocated = 1; \ - NAME = op; \ - } -#else -#define DEFINE_NAMED_STATIC_STRING(NAME, STR) \ - static PyObject* NAME = NULL; \ - if (NAME == NULL) { \ - PyObject* new_str = PyUnicode_FromString(STR); \ - new_str->ob_refcnt = 0x3fffffff; \ - NAME = new_str; \ +#define DEFINE_NAMED_STATIC_STRING(NAME, STR) \ + static PyObject* NAME = NULL; \ + if ((NAME) == NULL) { \ + NAME = Ci_InitStaticStringImpl((STR)); \ } -#endif // Shorter variant of DEFINE_NAMED_STATIC_STRING. #define DEFINE_STATIC_STRING(STR) DEFINE_NAMED_STATIC_STRING(s_##STR, (#STR)) + +// Helper for DEFINE_NAMED_STATIC_STRING. Do not call this directly. +PyObject* Ci_InitStaticStringImpl(const char* s); + +#ifdef __cplusplus +} // extern "C" +#endif diff --git a/cinderx/Common/type.cpp b/cinderx/Common/type.cpp index 3eb601e82..e8aed32a6 100644 --- a/cinderx/Common/type.cpp +++ b/cinderx/Common/type.cpp @@ -2,18 +2,18 @@ #include "cinderx/Common/type.h" -#if PY_VERSION_HEX >= 0x030C0000 #include "internal/pycore_typeobject.h" // @donotremove -#endif #include "cinderx/Common/dict.h" #include "cinderx/Common/log.h" #include "cinderx/Common/py-portability.h" #include "cinderx/Common/ref.h" #include "cinderx/Common/util.h" +#include "cinderx/Jit/compilation_lock.h" +#include "cinderx/Jit/threaded_compile.h" #include "cinderx/UpstreamBorrow/borrowed.h" -namespace jit { +namespace cinderx { std::string typeFullname(PyTypeObject* type) { PyObject* dict = _PyType_GetDict(type); @@ -25,30 +25,28 @@ std::string typeFullname(PyTypeObject* type) { return type->tp_name; } -#if PY_VERSION_HEX >= 0x030C0000 PyObject* getBorrowedTypeDictSafe(PyTypeObject* self) { - if (getThreadedCompileContext().compileRunning() && + if (jit::ThreadedCompileContext::compileRunning() && self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - PyInterpreterState* interp = getThreadedCompileContext().interpreter(); + PyInterpreterState* interp = jit::ThreadedCompileContext::interpreter(); managed_static_type_state* state = Cix_PyStaticType_GetState(interp, self); return state->tp_dict; } return getBorrowedTypeDict(self); } -#else -PyObject* getBorrowedTypeDictSafe(PyTypeObject* self) { - return getBorrowedTypeDict(self); -} -#endif BorrowedRef<> typeLookupSafe( BorrowedRef type, BorrowedRef<> name) { JIT_CHECK(PyUnicode_CheckExact(name), "name must be a str"); + +#if defined(Py_GIL_DISABLED) || CINDER_JIT_TSAN_ENABLED // Silence false positive from TSAN when checking Py_TPFLAGS_READY. - // This flag should never change during compliation although other - // flags may. - ThreadedCompileSerialize guard; + // This flag should never change during compilation although other + // flags may. We also need an attached thread state in free-threaded + // builds for the dict lookup. + jit::ThreadedCompileGILHolder guard; +#endif BorrowedRef mro{type->tp_mro}; for (size_t i = 0, n = PyTuple_GET_SIZE(mro); i < n; ++i) { @@ -63,22 +61,18 @@ BorrowedRef<> typeLookupSafe( if (BorrowedRef<> value{PyDict_GetItemWithError(dict, name)}) { return value; } - if constexpr (PY_VERSION_HEX < 0x030C0000) { - JIT_CHECK( - !PyErr_Occurred(), "Thread-unsafe exception during type lookup"); - } } return nullptr; } bool ensureVersionTag(BorrowedRef type) { - JIT_CHECK( - getThreadedCompileContext().canAccessSharedData(), - "Accessing type object needs lock"); + // We may be racing with the GIL here but we can tolerate it, we could + // invalidate the version tag after releasing the GIL anyway. if (Ci_Type_HasValidVersionTag(type)) { return true; } + jit::ThreadedCompileGILHolder lock; return PyUnstable_Type_AssignVersionTag(type); } -} // namespace jit +} // namespace cinderx diff --git a/cinderx/Common/type.h b/cinderx/Common/type.h index 5d3d32620..8243b810b 100644 --- a/cinderx/Common/type.h +++ b/cinderx/Common/type.h @@ -8,7 +8,7 @@ #include -namespace jit { +namespace cinderx { // When possible, return the fully qualified name of the given type (including // its module). Falls back to the type's bare name. @@ -29,4 +29,4 @@ BorrowedRef<> typeLookupSafe( // true if successful. bool ensureVersionTag(BorrowedRef type); -} // namespace jit +} // namespace cinderx diff --git a/cinderx/Common/util.cpp b/cinderx/Common/util.cpp index 2aebe6a0b..2172976a8 100644 --- a/cinderx/Common/util.cpp +++ b/cinderx/Common/util.cpp @@ -2,116 +2,36 @@ #include "cinderx/Common/util.h" -#if PY_VERSION_HEX >= 0x030C0000 -#include "internal/pycore_typeobject.h" // @donotremove -#endif - #include "cinderx/Common/log.h" #include "cinderx/Common/ref.h" +#include "cinderx/Jit/config.h" #include #include #include #include -static constexpr size_t INITIAL_SIZE = 104; - -struct jit_string_t { - char* str; - size_t capacity; - size_t pos; - char _string[INITIAL_SIZE]; -}; - -jit_string_t* ss_alloc() { - jit_string_t* ss = (jit_string_t*)malloc(sizeof(jit_string_t)); - - ss->capacity = INITIAL_SIZE; - ss->pos = 0; - ss->str = ss->_string; - return ss; -} - -void ss_free(jit_string_t* ss) { - if (ss->str != ss->_string) { - free(ss->str); - } - free(ss); -} - -void ss_reset(jit_string_t* ss) { - ss->pos = 0; -} - -const char* ss_get_string(const jit_string_t* ss) { - return ss->str; -} - -const char* ss_get_string(const auto_jit_string_t& ss) { - return ss_get_string(ss.get()); -} - -int ss_is_empty(const jit_string_t* ss) { - return ss->pos == 0; -} +namespace cinderx::jit { -int ss_vsprintf(jit_string_t* ss, const char* format, va_list args) { - while (1) { - int free_space = ss->capacity - ss->pos; +static_assert(kPyObjectPtrTag == 0); - va_list args_copy; - va_copy(args_copy, args); - int len = vsnprintf(ss->str + ss->pos, free_space, format, args_copy); - va_end(args_copy); - - if (free_space > len) { - ss->pos += len; - return len; - } - - if (ss->str != ss->_string) { - ss->capacity *= 2; - ss->str = (char*)realloc(ss->str, ss->capacity); - } else { - ss->capacity = 256; - ss->str = (char*)malloc(ss->capacity); - memcpy(ss->str, ss->_string, ss->pos); - } - JIT_CHECK( - ss->str != nullptr, - "Unable to allocate memory size = {} bytes", - ss->capacity); - } -} - -int ss_sprintf(jit_string_t* ss, const char* format, ...) { - va_list args; - va_start(args, format); - int n = ss_vsprintf(ss, format, args); - va_end(args); - return n; -} - -jit_string_t* ss_sprintf_alloc(const char* format, ...) { - jit_string_t* ss = ss_alloc(); - va_list args; - va_start(args, format); - ss_vsprintf(ss, format, args); - va_end(args); - return ss; -} +#ifdef Py_GIL_DISABLED +static_assert((kDeferredRcTag & kPyObjectTagBits) == kDeferredRcTag); +static_assert(std::has_single_bit(kDeferredRcTag)); +#endif -namespace jit { +} // namespace cinderx::jit -static bool s_use_stable_pointers{false}; +namespace cinderx { const void* getStablePointer(const void* ptr) { - return s_use_stable_pointers ? reinterpret_cast(0xdeadbeef) - : ptr; + return jit::getConfig().use_stable_pointers + ? reinterpret_cast(0xdeadbeef) + : ptr; } void setUseStablePointers(bool enable) { - s_use_stable_pointers = enable; + jit::getMutableConfig().use_stable_pointers = enable; } std::string unicodeAsString(PyObject* str) { @@ -128,4 +48,4 @@ Ref<> stringAsUnicode(std::string_view str) { return Ref<>::steal(PyUnicode_FromStringAndSize(str.data(), str.size())); } -} // namespace jit +} // namespace cinderx diff --git a/cinderx/Common/util.h b/cinderx/Common/util.h index 43f43436b..2f222134a 100644 --- a/cinderx/Common/util.h +++ b/cinderx/Common/util.h @@ -4,21 +4,26 @@ #include "cinderx/python.h" -#include -#include -#include +#ifdef Py_GIL_DISABLED +#include "internal/pycore_stackref.h" +#endif -#ifdef __cplusplus +#include "cinderx/Common/define.h" #include "cinderx/Common/log.h" +#include +#include #include #include #include #include +#include +#include #include #include #include #include +#include #include #include @@ -26,53 +31,19 @@ klass(const klass&) = delete; \ klass& operator=(const klass&) = delete -#define UNUSED __attribute__((unused)) +#define DISALLOW_MOVE_AND_ASSIGN(klass) \ + klass(klass&&) = delete; \ + klass& operator=(klass&&) = delete -extern "C" { -#endif - -struct jit_string_t* ss_alloc(void); -void ss_free(struct jit_string_t* ss); -void ss_reset(struct jit_string_t* ss); -int ss_is_empty(const struct jit_string_t* ss); -const char* ss_get_string(const struct jit_string_t* ss); -int ss_vsprintf(struct jit_string_t* ss, const char* format, va_list args); -int ss_sprintf(struct jit_string_t* ss, const char* format, ...); -struct jit_string_t* ss_sprintf_alloc(const char* format, ...); - -#ifdef __cplusplus -} - -constexpr bool kPyDebug = -#ifdef Py_DEBUG - true; -#else - false; -#endif - -constexpr bool kPyRefDebug = -#ifdef Py_REF_DEBUG - true; -#else - false; -#endif +#define UNUSED __attribute__((unused)) -constexpr bool kImmortalInstances = -#if defined(Py_IMMORTAL_INSTANCES) || PY_VERSION_HEX >= 0x030C0000 - true; -#else - false; +// This is for non-test builds. define FRIEND_TEST here so we don't have to +// include the googletest header in our headers to be tested. +#ifndef FRIEND_TEST +#define FRIEND_TEST(test_case_name, test_name) friend class test_case_name #endif -struct jit_string_deleter { - void operator()(jit_string_t* ss) const { - ss_free(ss); - } -}; - -using auto_jit_string_t = std::unique_ptr; - -const char* ss_get_string(const auto_jit_string_t& ss); +namespace cinderx { // Loading a method returns up to 2 items, for one of three possible outcomes: // * A callable plus an object instance (self). @@ -125,9 +96,75 @@ using GenResumeFunc = PyObject* (*)(PyObject * gen, namespace jit { +// Tagged PyObject values follow CPython's _PyStackRef tagging scheme in +// free-threaded builds. GIL builds keep using plain PyObject* values. +#ifdef Py_GIL_DISABLED +using TaggedPyObject = _PyStackRef; +constexpr uintptr_t kDeferredRcTag = Py_TAG_DEFERRED; +constexpr uintptr_t kPyObjectPtrTag = Py_TAG_PTR; +constexpr uintptr_t kPyObjectTagBits = Py_TAG_BITS; +#else +using TaggedPyObject = PyObject*; +constexpr uintptr_t kDeferredRcTag = 0; +constexpr uintptr_t kPyObjectPtrTag = 0; +constexpr uintptr_t kPyObjectTagBits = 0; +#endif + +// `kPyObjectPtrTag` being zero lets us treat an untagged PyObject* as a +// TaggedPyObject with no extra masking — see `untaggedPyObjectRef`. + +constexpr uint64_t kDeferredRcTagBit = + kFreeThreadedBuild ? std::countr_zero(kDeferredRcTag) : 0; + +inline uintptr_t taggedPyObjectBits(TaggedPyObject obj) { +#ifdef Py_GIL_DISABLED + return obj.bits; +#else + return reinterpret_cast(obj); +#endif +} + +inline bool isDeferredRcTagged(uint64_t raw) { + return kPyObjectTagBits != 0 && (raw & kPyObjectTagBits) == kDeferredRcTag; +} + +inline bool isDeferredRcTagged(TaggedPyObject obj) { + return isDeferredRcTagged(taggedPyObjectBits(obj)); +} + +inline uint64_t stripDeferredRcTag(uint64_t raw) { + return raw & ~static_cast(kPyObjectTagBits); +} + +inline PyObject* untaggedPyObject(TaggedPyObject obj) { + return reinterpret_cast( + stripDeferredRcTag(taggedPyObjectBits(obj))); +} + +inline TaggedPyObject taggedPyObject( + PyObject* obj, + [[maybe_unused]] uintptr_t tag) { +#ifdef Py_GIL_DISABLED + return {reinterpret_cast(obj) | tag}; +#else + return obj; +#endif +} + +inline TaggedPyObject untaggedPyObjectRef(PyObject* obj) { + return taggedPyObject(obj, kPyObjectPtrTag); +} + +inline TaggedPyObject addDeferredRcTag(PyObject* obj) { + return taggedPyObject(obj, kDeferredRcTag); +} + +} // namespace jit + constexpr int kPointerSize = sizeof(void*); constexpr size_t kStackAlign = 16; +constexpr size_t kVecDSize = 16; constexpr int kKiB = 1024; constexpr int kMiB = kKiB * kKiB; @@ -145,20 +182,34 @@ constexpr bool isPowerOfTwo(T x) { } template + requires std::is_integral_v constexpr T roundDown(T x, size_t n) { if (n == 0) { return n; } + JIT_DCHECK(isPowerOfTwo(n), "Must be 0 or a power of 2"); return (x & -n); } template + requires std::is_integral_v constexpr T roundUp(T x, size_t n) { if (n == 0) { - return n; + return T{0}; } - return roundDown(x + n - 1, n); + + JIT_DCHECK(isPowerOfTwo(n), "Must be 0 or a power of 2"); + + using UnsignedT = std::make_unsigned_t; + constexpr auto max = static_cast(std::numeric_limits::max()); + JIT_CHECK(n - 1 <= max, "roundUp overflow"); + + const auto mask = static_cast(n - 1); + const auto value = static_cast(x); + JIT_CHECK(value <= max - mask, "roundUp overflow"); + + return static_cast((value + mask) & ~mask); } template @@ -204,18 +255,6 @@ std::string unicodeAsString(PyObject* str); // error. Ref<> stringAsUnicode(std::string_view str); -inline int popcount(unsigned i) { - return __builtin_popcount(i); -} - -inline int popcount(unsigned long i) { - return __builtin_popcountl(i); -} - -inline int popcount(unsigned long long i) { - return __builtin_popcountll(i); -} - // Look up an item in the given map. Always abort if key doesn't exist. template auto& map_get_strict(M& map, const K& key) { @@ -297,6 +336,27 @@ bool fitsSignedInt(T val) { return fitsSignedInt(reinterpret_cast(val)); } +inline void* malloc_aligned(size_t size, size_t alignment) { +#ifdef WIN32 + return _aligned_malloc(size, alignment); +#else + void* chunk = nullptr; + int result = posix_memalign(&chunk, alignment, size); + if (result) { + return nullptr; + } + return chunk; +#endif +} + +inline void free_aligned(void* ptr) { +#ifdef WIN32 + _aligned_free(ptr); +#else + free(ptr); +#endif +} + // std::unique_ptr for objects created with std::malloc() rather than new. struct FreeDeleter { void operator()(void* ptr) const { @@ -305,7 +365,6 @@ struct FreeDeleter { }; template using unique_c_ptr = std::unique_ptr; - template class ScopeExit { public: @@ -343,9 +402,38 @@ class CriticalSectionGuard final { #endif }; +// Typed, cross-version equivalent of CPython's FT_ATOMIC_LOAD_PTR_ACQUIRE(). +template +T* ftAtomicLoadPtrAcquire(T*& ptr) noexcept { + if constexpr (kFreeThreadedBuild) { +#ifdef __cpp_lib_atomic_ref + return std::atomic_ref(ptr).load(std::memory_order_acquire); +#else + return __atomic_load_n(&ptr, __ATOMIC_ACQUIRE); +#endif + } else { + return ptr; + } +} + +// Typed, cross-version equivalent of CPython's +// FT_ATOMIC_STORE_PTR_RELAXED(). +template +void ftAtomicStorePtrRelaxed(T*& ptr, T* value) noexcept { + if constexpr (kFreeThreadedBuild) { +#ifdef __cpp_lib_atomic_ref + std::atomic_ref(ptr).store(value, std::memory_order_relaxed); +#else + __atomic_store(&ptr, &value, __ATOMIC_RELAXED); +#endif + } else { + ptr = value; + } +} + #define SCOPE_EXIT_INTERNAL2(lname, aname, ...) \ auto lname = [&]() { __VA_ARGS__; }; \ - jit::ScopeExit aname(std::move(lname)); + cinderx::ScopeExit aname(std::move(lname)); #define SCOPE_EXIT_TOKENPASTE(x, y) SCOPE_EXIT_##x##y @@ -357,125 +445,23 @@ class CriticalSectionGuard final { #define SCOPE_EXIT(...) SCOPE_EXIT_INTERNAL1(__COUNTER__, __VA_ARGS__) -// Return a crc32 checksum of the bytecode for the given code object. -// A frozen list is effectively a vector that is dynamically allocated at -// runtime, but then can no longer be resized. -template -class FrozenList { - public: - FrozenList() = default; - - // Make FrozenList copy constructible. - FrozenList(const FrozenList& other) { - reserve(other.size_); - std::copy(other.begin(), other.end(), ptr_.get()); - } - - // Make FrozenList move constructible. - FrozenList(FrozenList&& other) noexcept { - *this = std::move(other); - } - - // Make FrozenList move assignable. - FrozenList& operator=(FrozenList&& other) noexcept { - if (this != &other) { - ensureUninitialized(); - - size_ = other.size_; - ptr_ = std::move(other.ptr_); - - other.size_ = 0; - other.ptr_ = nullptr; - } - - return *this; - } - - // Construct a frozen list from the given initializer list. - /* implicit */ FrozenList(std::initializer_list values) { - reserve(values.size()); - std::copy(values.begin(), values.end(), ptr_.get()); - } - - // Make FrozenList copy assignable. - FrozenList& operator=(const FrozenList& other) { - if (this != &other) { - reserve(other.size_); - std::copy(other.begin(), other.end(), ptr_.get()); - } - - return *this; - } - - // Destroy a frozen list. - ~FrozenList() = default; - - // The size of the list. - size_t size() const { - return size_; - } - - // Set the size of the frozen list and build a new pointer to the data, then - // fill the data with the default value for the type. - // - // In order to call this function, T must be default constructible. - void resize(size_t size) { - resize(size, T{}); - } - - // Set the size of the frozen list and build a new pointer to the data, then - // fill the data with a copy of the given value. - // - // In order to call this function, T must be copy constructible. - void resize(size_t size, const T& val) { - reserve(size); - std::fill(ptr_.get(), ptr_.get() + size, val); - } - - // Provide the begin function for immutable range-based for-loop support. - const T* begin() const { - return ptr_.get(); - } - - // Provide the end function for immutable range-based for-loop support. - const T* end() const { - return ptr_.get() + size_; - } - - // Provide the [] operator for accessing elements by index. - T& operator[](size_t index) const { - return ptr_[index]; - } - - // Like the [] operator, but throws an exception if the index is out of range. - T& at(size_t index) const { - if (index >= size_) { - throw std::out_of_range("Index out of range"); - } - return ptr_[index]; - } - - private: - size_t size_{0}; - std::unique_ptr ptr_; - - // Raise an exception if the list has already been initialized. - void ensureUninitialized() { - if (ptr_ != nullptr) { - throw std::runtime_error("Cannot resize a frozen list twice"); - } - } - - // Set the size of the frozen list and build a new pointer to the data. - void reserve(size_t size) { - ensureUninitialized(); - size_ = size; - - if (size != 0) { - ptr_ = std::make_unique(size); - } - } -}; +// Relaxed atomic store for func->vectorcall for thread-safe writes under +// free-threading and to satisfy TSAN. A release store might be the right +// choice in some cases to publish JIT metadata to readers, but CPython's +// _PyVectorcall_FunctionInline does a plain (non-acquire) load, so +// release/acquire isn't achievable without CPython changes. +// Under the GIL this is unnecessary, but relaxed has no overhead so we skip +// the Py_GIL_DISABLED guard. +inline void setVectorcall( + BorrowedRef func, + vectorcallfunc entry) { +#ifdef __cpp_lib_atomic_ref + std::atomic_ref(func->vectorcall) + .store(entry, std::memory_order_relaxed); +#else + __atomic_store_n(&func->vectorcall, entry, __ATOMIC_RELAXED); +#endif +} using FuncVisitor = void (*)(BorrowedRef); @@ -491,23 +477,4 @@ inline void walkFunctionObjects(FuncVisitor visitor) { PyUnstable_GC_VisitObjects(wrapper, reinterpret_cast(visitor)); } -} // namespace jit - -template -inline constexpr D bit_cast(const S& src) { - static_assert(sizeof(S) == sizeof(D), "src and dst must be the same size"); - static_assert( - std::is_scalar_v && std::is_scalar_v, - "both src and dst must be of scalar type."); - D dst; - std::memcpy(&dst, &src, sizeof(dst)); - return dst; -} - -#endif - -// this is for non-test builds. define FRIEND_TEST here so we don't -// have to include the googletest header in our headers to be tested. -#ifndef FRIEND_TEST -#define FRIEND_TEST(test_case_name, test_name) friend class test_case_name -#endif +} // namespace cinderx diff --git a/cinderx/Common/watchers.cpp b/cinderx/Common/watchers.cpp index cfdb4d6c9..4db3d57cb 100644 --- a/cinderx/Common/watchers.cpp +++ b/cinderx/Common/watchers.cpp @@ -2,6 +2,8 @@ #include "cinderx/Common/watchers.h" +#include "cinderx/Common/util.h" + namespace cinderx { WatcherState::WatcherState() = default; @@ -75,10 +77,14 @@ void WatcherState::setTypeWatcher(TypeWatcher watcher) { } int WatcherState::watchDict(BorrowedRef dict) { + // TODO: Remove once CPython's dict watcher API is thread-safe. + cinderx::CriticalSectionGuard guard(dict); return PyDict_Watch(dict_watcher_id_, dict); } int WatcherState::unwatchDict(BorrowedRef dict) { + // TODO: Remove once CPython's dict watcher API is thread-safe. + cinderx::CriticalSectionGuard guard(dict); return PyDict_Unwatch(dict_watcher_id_, dict); } diff --git a/cinderx/Common/weakref_helpers.c b/cinderx/Common/weakref_helpers.c new file mode 100644 index 000000000..13bf7c76a --- /dev/null +++ b/cinderx/Common/weakref_helpers.c @@ -0,0 +1,17 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#include "cinderx/Common/py-portability.h" + +#if PY_VERSION_HEX >= 0x030E0000 +#include "internal/pycore_weakref.h" +#endif + +void Ci_ClearWeakRefs(PyObject* self, PyObject* weakrefs) { +#if PY_VERSION_HEX >= 0x030E0000 + FT_CLEAR_WEAKREFS(self, weakrefs); +#else + if (weakrefs != NULL) { + PyObject_ClearWeakRefs(self); + } +#endif +} diff --git a/cinderx/Docs/README.md b/cinderx/Docs/README.md deleted file mode 100644 index 3de0ab95a..000000000 --- a/cinderx/Docs/README.md +++ /dev/null @@ -1 +0,0 @@ -Internal developer documentation for CinderX and Static Python. diff --git a/cinderx/Docs/StrictModules/README.md b/cinderx/Docs/StrictModules/README.md deleted file mode 100644 index 5372f34b4..000000000 --- a/cinderx/Docs/StrictModules/README.md +++ /dev/null @@ -1,32 +0,0 @@ -# Strict Modules - -> [*Python is more dynamic than the language I \*wanted\* to design. — Guido van Rossum*](https://mail.python.org/pipermail/python-list/2001-August/073435.html) - - -## What are strict modules? - -Strict modules are an opt-in mechanism for restricting the dynamism of -top-level (module-level) code in a Python module. These stricter -semantics are designed to have multiple benefits: - -* Eliminate common classes of developer errors -* Improve the developer experience -* Unlock new opportunities for optimizing Python code and simplify other classes of optimizations - -The strict module analyzer is no longer supported after 3.10. In 3.12 the only impact -of strict modules is that modules marked strict are immutable and their types -are frozen at runtime after the module definition completes. - -In 3.10 while strict modules alter what you can do at the top-level of your code -they don't limit what you can do inside of your function definitions. They -also are designed to not limit the expressiveness of what you can do even -at the top-level. The limits on strict modules at the top-level all have a single -goal: Make sure that module definitions can reliably be statically analyzed. - -In order to support this there are also some runtime changes for strict -modules, mostly around immutability. In order to be able to make guarantees -about the analysis we must be certain that it won't later be invalidated by -runtime changes. To this end strict modules themselves are immutable and -types defined within strict modules are immutable as well. - -## Overview diff --git a/cinderx/Docs/StrictModules/guide/conversion/class_inst_conflict.rst b/cinderx/Docs/StrictModules/guide/conversion/class_inst_conflict.rst deleted file mode 100644 index 03b6fd096..000000000 --- a/cinderx/Docs/StrictModules/guide/conversion/class_inst_conflict.rst +++ /dev/null @@ -1,48 +0,0 @@ -Class / Instance Conflict -######################### - - -One of the changes that strict modules introduces is the promotion of instance -members to being class level declarations. For more information on this pattern -see :doc:`../limitations/class_attrs`. - -A typical case for this is when you'd like to have a default method implementation -but override it on a per instance basis: - -.. code-block:: python - - class C: - def f(self): - return 42 - - a = C() - a.f = lambda: "I'm a special snowflake" - - -If you attempt this inside of a strict module you'll get an AttributeError that -says "'C' object attribute 'f' is read-only". This is because the instance -doesn't have any place to store the method. You might think that you can declare -the field explicitly as specified in the documentation: - -.. code-block:: python - - class C: - f: ... - def f(self): - return 42 - -But instead you'll get an error reported by strict modules stating that there's -a conflict with the variable. To get around this issue you can promote the function -to always be treated as an instance member: - -.. code-block:: python - - class C: - def __init__(self): - self.f = self.default_f - - def default_f(self): - return 42 - - a = C() - a.f = lambda: "I'm a special snowflake" # Ok, you are a special snowflake diff --git a/cinderx/Docs/StrictModules/guide/conversion/external_modification.rst b/cinderx/Docs/StrictModules/guide/conversion/external_modification.rst deleted file mode 100644 index adbbb436b..000000000 --- a/cinderx/Docs/StrictModules/guide/conversion/external_modification.rst +++ /dev/null @@ -1,53 +0,0 @@ -Modifying External State -######################## - -Strict modules enforces object :doc:`ownership `, -and will not allow module-level code to modify any object defined -in a different module. - -One common example of this is to have a global registry of some sort of -objects: - -**methods.py** - -.. code-block:: python - - import __strict__ - - ROUTES = list() - - def route(f): - ROUTES.append(f) - return f - -**routes.py** - -.. code-block:: python - - import __strict__ - - from methods import route - - @route - def create_user(*args): - ... - - -Here we have one module which is maintaining a global registry, which is -populated as a side effect of importing another module. If for some reason -one module doesn't get imported or if the order of imports changes then the -program's execution can change. When strict modules analyzes this code it will -report a :doc:`/strict_modules/guide/errors/modify_imported_value`. - -A better pattern for this is to explicitly register the values in a central -location: - -**methods.py** - -.. code-block:: python - - import __strict__ - - from routes import create_user - - ROUTES = [create_user, ...] diff --git a/cinderx/Docs/StrictModules/guide/conversion/index.rst b/cinderx/Docs/StrictModules/guide/conversion/index.rst deleted file mode 100644 index fd90b9d28..000000000 --- a/cinderx/Docs/StrictModules/guide/conversion/index.rst +++ /dev/null @@ -1,12 +0,0 @@ -Conversion Tips -############### - -This section of the documentation includes common patterns that violate the -limitations of strict modules and solutions you can use to work around them. - -.. toctree:: - :maxdepth: 1 - :titlesonly: - :glob: - - * diff --git a/cinderx/Docs/StrictModules/guide/conversion/loose_slots.rst b/cinderx/Docs/StrictModules/guide/conversion/loose_slots.rst deleted file mode 100644 index 3842b805d..000000000 --- a/cinderx/Docs/StrictModules/guide/conversion/loose_slots.rst +++ /dev/null @@ -1,42 +0,0 @@ -The @loose_slots decorator -########################## - -Instances of strict classes have `__slots__ -`_ automatically -created for them. This means they will raise ``AttributeError`` if you try to -add any attribute to them that isn't declared with a type annotation on the -class itself (e.g. ``attrname: int``) or assigned in the ``__init__`` method. - -When initially converting a module to strict, if it is widely-used it can be -hard to verify that there isn't code somewhere tacking extra attributes onto -instances of classes defined in that module. In this case, you can temporarily -place the ``strict_modules.loose_slots`` decorator on the class for a safer -transition. Example: - -.. code-block:: python - - import __strict__ - - from cinderx.compiler.strict.runtime import loose_slots - - @loose_slots - class MyClass: - ... - -This decorator will allow extra attributes to be added to the class, but will -fire a warning when it happens. You can access these warnings by setting a -warnings callback function: - -.. code-block:: python - - from cinder import cinder_set_warnings_handler - - def log_cinder_warning(msg: str, *args: object) -> None: - # ... - - cinder_set_warnings_handler(log_cinder_warning) - -Typically you'd want to set a warnings handler that logs these warnings somewhere, -then you can deploy some new strict modules using `@loose_slots`, -and once the code has been in production for a bit and you see no warnings -fired, you can safely remove `@loose_slots`. diff --git a/cinderx/Docs/StrictModules/guide/conversion/module_access.rst b/cinderx/Docs/StrictModules/guide/conversion/module_access.rst deleted file mode 100644 index 17951e1e7..000000000 --- a/cinderx/Docs/StrictModules/guide/conversion/module_access.rst +++ /dev/null @@ -1,27 +0,0 @@ -Top-level Module Access -####################### - -A common pattern is to import a module and access members from that module: - -.. code-block:: python - - from useful import submodule - - class MyClass(submodule.BaseClass): - pass - -If “submodule” is not strict, then we don't know what it is and what side -effects could happen by dotting through it. So this pattern is disallowed -inside of a strict module when importing from a non-strict module. Instead -you can transform the code to: - -.. code-block:: python - - from useful.submodule import BaseClass - - class MyClass(BaseClass): - pass - -This will cause any side effects that are possible to occur only when -the non-strict module is imported; the execution of the rest of the -strict module will be known to be side effect free. diff --git a/cinderx/Docs/StrictModules/guide/conversion/singletons.rst b/cinderx/Docs/StrictModules/guide/conversion/singletons.rst deleted file mode 100644 index 1b6e29424..000000000 --- a/cinderx/Docs/StrictModules/guide/conversion/singletons.rst +++ /dev/null @@ -1,66 +0,0 @@ -Global Singletons -################# - -Sometimes it might be useful to encapsulate a set of functionality into -a class and then have a global singleton of that class. And sometimes -that global singleton might have dependencies on non-strict code which -makes it impossible to construct at the top-level in a strict module. - -.. code-block:: python - - from non_strict import get_counter_start - - class Counter: - def __init__(self) -> None: - self.value: int = get_counter_start() - - def next(self) -> int: - res = self.value - self.value += 1 - return res - - COUNTER = Counter() - -One way to address this is to refactor the Counter class so that it -does less when constructed, delaying some work until first use. For -example: - -.. code-block:: python - - from non_strict import get_counter_start - - class Counter: - def __init__(self) -> None: - self.value: int = -1 - - def next(self) -> int: - if self.value == -1: - self.value = get_counter_start() - res = self.value - self.value += 1 - return res - COUNTER = Counter() - -Another approach is that instead of constructing the singleton at the -top of the file you can push this into a function so it gets defined -the first time it'll need to be used: - -.. code-block:: python - - _COUNTER = None - - def get_counter() -> Counter: - global _COUNTER - if _COUNTER is None: - _COUNTER = Counter() - return _COUNTER - -You can also use an lru_cache instead of a global variable: - -.. code-block:: python - - from functools import lru_cache - - @lru_cache(maxsize=1) - def get_counter() -> Counter: - return Counter() diff --git a/cinderx/Docs/StrictModules/guide/conversion/splitting_modules.rst b/cinderx/Docs/StrictModules/guide/conversion/splitting_modules.rst deleted file mode 100644 index 265024c2a..000000000 --- a/cinderx/Docs/StrictModules/guide/conversion/splitting_modules.rst +++ /dev/null @@ -1,69 +0,0 @@ -Splitting Modules -################# - -Sometimes a module might contain functionality which is dependent upon certain -behavior which cannot be analyzed - either it truly has external side effects, -it is dependent upon another module which cannot yet be strictified and needs -to be used at the top-level, or it is dependent upon something which strict -modules have not yet been able to analyze. - -In these cases one possible solution, although generally a last resort, -is to break the module into two modules. The first module will only contain -the code which cannot be safely strictified. The second module will contain -all of the code that can be safely treated as strict. A better way to do this -is to not have the unverifable code happen at startup, but if that's not -possible then splitting is an acceptable option. - -Because strict modules can still import non-strict modules the strict module -can continue to expose the same interface as it previously did, and no other -code needs to be updated. The only limitation to this is that it requires -that the module being strictified doesn't need to interact with the non-strict -elements at the top level. For example classes could still create instances -of them, but the strict module couldn't call functions in the non-strict -module at the top level. - - -.. code-block:: python - - import csv - from random import choice - - FAMOUS_PEOPLE = list(csv.reader(open('famous_people.txt').readlines())) - - class FamousPerson: - def __init__(self, name, age, height): - self.name = name - self.age = int(age) - self.height = float(height) - - def get_random_person(): - return FamousPerson(*choice(FAMOUS_PEOPLE)) - - -We can split this into two modules, one which does the unverifable read of our -sample data from disk and another which returns the random piece of sample data: - - -.. code-block:: python - - import csv - - FAMOUS_PEOPLE = list(csv.reader(open('famous_people.txt').readlines())) - - -And we can have another module which exports our FamousPerson class along with -the API to return a random famous person: - -.. code-block:: python - - from random import choice - from famous_people_data import FAMOUS_PEOPLE - - class FamousPerson: - def __init__(self, name, age, height): - self.name = name - self.age = int(age) - self.height = float(height) - - def get_random_person(): - return FamousPerson(*choice(FAMOUS_PEOPLE)) diff --git a/cinderx/Docs/StrictModules/guide/conversion/stubs.rst b/cinderx/Docs/StrictModules/guide/conversion/stubs.rst deleted file mode 100644 index 4e5883649..000000000 --- a/cinderx/Docs/StrictModules/guide/conversion/stubs.rst +++ /dev/null @@ -1,67 +0,0 @@ -Strict Module Stubs -################### - -Sometimes your modules depend on other modules that cannot be directly -strictified - it could depend on a Cython module, or a module from a -third-party library whose source code you can't modify. - -In this situation, if you are certain that the dependency is strict, you -can provide a strict module stub file (`.pys`) describing the behavior of -the module. Put the strict module stub file in your strict module stubs directory -(this is configured via `-X strict-module-stubs-path=...=` or -`PYTHONSTRICTMODULESTUBSPATH` env var, or by subclassing `StrictSourceFileLoader` -and passing a `stub_path` argument to `super().__init__(...)`.) - -There are two ways to stub a class or function in a strict module stub file. -You can provide a full Python implementation, which is useful in the case -of stubbing a Cython file, or you can just provide a function/class name, -with a `@implicit` decorator. In the latter case, the stub triggers the -strict module analyzer to look for the source code on `sys.path` and analyze -the source code. - -If the module you depend on is already actually strict-compliant you can -simplify the stub file down to just contain the single line `__implicit__`, -which just says "go use the real module contents, they're fine". See -`cinderx/PythonLib/cinderx/compiler/strict/stubs/_collections_abc.pys` for an -existing example. Per-class/function stubs are only needed where the stdlib -module does non-strict things at module level, so we need to extract just the -bits we depend on and verify them for strictness. - -If both a `.py` file and a `.pys` file exist, the strict module analyzer will -prioritize the `.pys` file. This means adding stubs to existing -modules in your codebase will shadow the actual implementation. -You should probably avoid doing this. - -Example of Cython stub: - -**myproject/worker.py** - -.. code-block:: python - - from some_cython_mod import plus1 - - two = plus1(1) - - -Here you can provide a stub for the Cython implementation of `plus1` - -**strict_modules/stubs/some_cython_mod.pys** - -.. code-block:: python - - # a full reimplementation of plus1 - def plus1(arg): - return arg + 1 - -Suppose you would like to use the standard library functions `functools.wraps`, -but the strict module analysis does not know of the library. You can add an implicit -stub: - -**strict_modules/stubs/functools.pys** - -.. code-block:: python - - @implicit - def wraps(): ... - -You can mix explicit and implicit stubs. See `CinderX/cinderx/compiler/strict/stubs` for some examples. diff --git a/cinderx/Docs/StrictModules/guide/conversion/testing.rst b/cinderx/Docs/StrictModules/guide/conversion/testing.rst deleted file mode 100644 index d91ed9d49..000000000 --- a/cinderx/Docs/StrictModules/guide/conversion/testing.rst +++ /dev/null @@ -1,32 +0,0 @@ -Testing -####### - -You might be wondering how you're going to go and mock out functionality in -a strict module when we've already asserted that strict modules are immutable. -While we certainly don't want you to modify strict modules in production -they can be monkey patched in testing scenarios! - -To enable patching strict modules during testing, you will need to customize -your strict loader (see :doc:`../quickstart`) by creating a subclass and -passing `enable_patching = True` before installing the loader. - -.. code-block:: python - - from cinderx.compiler.strict.loader import StrictSourceFileLoader - from typing import final - - @final - class StrictSourceFileLoaderWithPatching(StrictSourceFileLoader): - def __init__(self) -> None: - # ... - super().__init__( - # ... - enable_patching = True, - # ... - ) - -With patching enabled, you will be able to patch symbols in strict modules: - -.. code-block:: python - - mystrictmodule.patch("name", new_value) diff --git a/cinderx/Docs/StrictModules/guide/errors/class_attr_conflict.rst b/cinderx/Docs/StrictModules/guide/errors/class_attr_conflict.rst deleted file mode 100644 index b8be8ebc8..000000000 --- a/cinderx/Docs/StrictModules/guide/errors/class_attr_conflict.rst +++ /dev/null @@ -1,49 +0,0 @@ -ClassAttributesConflictException -################################ - - Class member conflicts with instance member: foo - - -Strict modules require that instance attributes are distinct from class level -attributes such as methods. - -.. code-block:: python - - class C: - def __init__(self, flag: bool): - if flag: - self.f = lambda: 42 - - def f(self): - return '42' - - -``ClassAttributesConflictException`` 'Class member conflicts with instance member: f' - -In this example we are attempting to override a method defined on the class -with a unique per-instance method. We cannot do this in a strict module -because the instance attribute is actually defined at the class level. - - -.. code-block:: python - - class C: - value = None - def __init__(self, flag: bool): - if flag: - self.value = 42 - - - -``ClassAttributesConflictException`` 'Class member conflicts with instance member: value' - -In this example we're attempting to provide a fallback value that's declared -at the class level. - -In both of these cases the solution is to define the value either -completely at the instance or class level. For example we could change the -first example to always set `self.f` in the constructor, just sometimes setting -it to the default value. - -For additional information see the section on -:doc:`/strict_modules/guide/limitations/class_attrs`. diff --git a/cinderx/Docs/StrictModules/guide/errors/import_star_disallowed.rst b/cinderx/Docs/StrictModules/guide/errors/import_star_disallowed.rst deleted file mode 100644 index 9491cb1b8..000000000 --- a/cinderx/Docs/StrictModules/guide/errors/import_star_disallowed.rst +++ /dev/null @@ -1,23 +0,0 @@ -ImportStarDisallowedException -############################# - - Strict modules may not import ``*``. - -This error indicates that you are attempting to do a ``from module import *``. - -.. code-block:: python - - from foo import * - - -Strict modules simply outright prohibit this construct. Import stars are not -only generally considered bad style but they also make it impossible to -understand what attributes are defined within a module. Because import * can -bring in any name it has the possibility of overwriting existing names that -have been previously imported. - -To work around this explicitly import the values that you intend to use from -the module. - -For additional information see the section on -:doc:`/strict_modules/guide/limitations/imports`. diff --git a/cinderx/Docs/StrictModules/guide/errors/index.rst b/cinderx/Docs/StrictModules/guide/errors/index.rst deleted file mode 100644 index 6510522f7..000000000 --- a/cinderx/Docs/StrictModules/guide/errors/index.rst +++ /dev/null @@ -1,61 +0,0 @@ -Intro to Errors -############### - -First, read the error and the documentation page linked from the error -message to understand what it's telling you! - -One common theme in all of these error messages will be the concept of an -"unknown value". An unknown value in a strict module is a value that comes -from a source that the strict module doesn't understand. That typically means -it's imported from a non-strict module or from an operation on an unknown -value. The error message will attempt to give you detailed information on the -source of the value, including the initial unknown name that was imported or -used and the chain of operations performed against that value. - -Typically an unknown value will be displayed as one of ```` where unknown is the name of the non-strict imported module, -```` where the value comes from ``from -unknown import some_name``, or just a simple name like ```` if the -value comes from an undefined global or unknown built-in. - -There are a few basic causes of errors: - -* Your code is actually doing something side-effecty or unsafe in top-level - code (including functions called from top-level code, e.g. decorators), and - strict modules is alerting you to the problem. In this case you should adjust - your code to not do this. :doc:`../conversion/singletons` has some advice on - moving side-effecty code out of the import codepath by making it lazy. - -* Your code is actually fine, but you are using at module level - some class or function that you imported from a non-strict module, so our - analysis doesn't know about it and is flagging that you are using an "unknown - value" at import time. Early in adoption, this will likely be a common case. - Options for dealing with it: - - a. You can try converting the module you are importing from to be strict - itself. If the module is hard to convert, but the specific piece of it you - need is not, you could :doc:`split the module - <../conversion/splitting_modules>`. - b. If the dependency is external to your codebase (i.e. third-party), - you can add a :doc:`stub file <../conversion/stubs>` to tell - strict modules about it. - c. If (a) and (b) are hard and you need to unblock yourself, you can - remove `import __strict__` from your module for now. This could require - de-strictifying other modules as well, if other strict modules import from - yours. - -* Your code is actually fine, but strict modules is not able to analyze it - correctly (e.g. a missing built-in, or some aspect of the language we aren't - fully analyzing correctly yet). In this case you should just remove - `import __strict__` to unblock yourself and report a bug so we - can fix the problem. - -For guidance on specific errors please refer to this list: - - -.. toctree:: - :maxdepth: 1 - :titlesonly: - :glob: - - * diff --git a/cinderx/Docs/StrictModules/guide/errors/modify_imported_value.rst b/cinderx/Docs/StrictModules/guide/errors/modify_imported_value.rst deleted file mode 100644 index a5712f118..000000000 --- a/cinderx/Docs/StrictModules/guide/errors/modify_imported_value.rst +++ /dev/null @@ -1,15 +0,0 @@ -StrictModuleModifyImportedValueException -######################################## - - from module is modified by ; this is - prohibited. - -Strict modules only allow a module to modify values that are defined/created -within the defining module. This prevents one module from having side effects -that would impact another module or non-determinism based upon the order of -imports. - -For additional information see the section on -:doc:`/strict_modules/guide/limitations/ownership`. - -For guidance on how to fix this see :doc:`/strict_modules/guide/conversion/external_modification`. diff --git a/cinderx/Docs/StrictModules/guide/errors/prohibited_callable.rst b/cinderx/Docs/StrictModules/guide/errors/prohibited_callable.rst deleted file mode 100644 index 576f58cfd..000000000 --- a/cinderx/Docs/StrictModules/guide/errors/prohibited_callable.rst +++ /dev/null @@ -1,25 +0,0 @@ -ProhibitedBuiltinException -########################## - - Call to built-in '' is prohibited at module level. - -This error indicates that you are attempting to call a known built-in function -which strict modules do not support. - -Currently this error applies to calling exec and eval. - - -.. code-block:: python - - exec('x = 42') - - -Currently strict modules do not support exec or eval at the top level. If you -must use them you can move them into a function which lazily computes the value -and stores it in a global variable. See :doc:`../conversion/singletons` as -one possible solution to this. - -In the future strict modules may support exec/eval as long as the values being -passed to them are deterministic. This would enable more complex library code -to be defined within strict modules. One example of this in real-world code -is Python's namedtuple class which uses exec to define tuple instances. diff --git a/cinderx/Docs/StrictModules/guide/errors/unknown_call.rst b/cinderx/Docs/StrictModules/guide/errors/unknown_call.rst deleted file mode 100644 index f188bf6ee..000000000 --- a/cinderx/Docs/StrictModules/guide/errors/unknown_call.rst +++ /dev/null @@ -1,65 +0,0 @@ -UnknownValueCallException -######################### - - Module-level call of non-strict value 'function()' is prohibited. - -This error indicates that you are attempting to call a value at module level -which the strict modules analysis does not understand. - -Currently the most likely reason you would see this error is because you are -importing something from a non-strict module and then calling it at module -level. For example: - -.. code-block:: python - - import __strict__ - from nonstrict_module import something - - x = something() - - -This code will result in an error message such as: - -``UnknownValueCallException`` 'Call of unknown value 'something()' is prohibited.' - -If `something()` has no side effects (it only returns a value), your options, -in order of preference, are to a) strictify the `nonstrict_module`, b) call -`something()` lazily on demand rather than at module level, or c) -de-strictify the module you are currently working in. If `something()` does -in fact have side effects, the only option you should consider is (b). - -Another case where you might see this is with an unsupported builtin: - -.. code-block:: python - - import __strict__ - - print('hi') - - -In this case we're attempting to call a built-in function which strict -modules don't support. Printing is typically a side effect so we don't -currently support it at module level. There are a number of other built-ins -which aren't currently supported as well. For the full list of supported -builtins see :doc:`/strict_modules/guide/limitations/builtins`. - -We can fix this case by removing the usage of the built-in at the top-level. - -If the function is a built-in and something you think strict modules should -support at module level, you might want to report a bug! - -.. code-block:: python - - import __strict__ - - ALPHABETE = lsit('abcdefghijklmnopqrstuvwxyz') - - -``UnknownValueCallException`` 'Call of unknown value 'lsit()' is prohibited.' - -In this case we've simply made a mistake and misspelled the normal built-in -list. One nice thing is that strict modules will detect this and give you -an early warning. We can fix it by just fixing the spelling. - -You can look at :ref:`conversion_tips` for other possible solutions to issues -like this. diff --git a/cinderx/Docs/StrictModules/guide/errors/unknown_value_attribute.rst b/cinderx/Docs/StrictModules/guide/errors/unknown_value_attribute.rst deleted file mode 100644 index b01ab4221..000000000 --- a/cinderx/Docs/StrictModules/guide/errors/unknown_value_attribute.rst +++ /dev/null @@ -1,41 +0,0 @@ -UnknownValueAttributeException -############################## - - Module-level attribute access on non-strict value 'value.attr' is prohibited. - -This error indicates that you are attempting to access an attribute on a -value that can't be analyzed by strict modules (e.g. because it is imported -from a non-strict module). Because attribute access in Python can execute -arbitrary code, doing this on an unknown value (where we can't analyze the -effects) could cause arbitrary side effects and is prohibited at module -level. - -.. code-block:: python - - from nonstrict import something - - class MyClass(something.SomeClass): - pass - -This code will result in an error message such as: - -``UnknownValueAttributeException`` '.SomeClass' - -The error tells you both what unknown value you are accessing an attribute -on, and the name of the attribute. - -One possible solution to this is making the nonstrict module strict so that -``something`` can be used at the top-level. If the nonstrict module is not in -your codebase you could create a :doc:`stub file <../conversion/stubs>` for it. - -In a case like the above example where the unknown value is an imported module, -you can also solve it like this: - -.. code-block:: python - - from nonstrict.something import SomeClass - - class MyClass(SomeClass): - pass - -You can look at :ref:`conversion_tips` for more ways to fix this error. diff --git a/cinderx/Docs/StrictModules/guide/errors/unknown_value_binary_op.rst b/cinderx/Docs/StrictModules/guide/errors/unknown_value_binary_op.rst deleted file mode 100644 index 7ca0aec08..000000000 --- a/cinderx/Docs/StrictModules/guide/errors/unknown_value_binary_op.rst +++ /dev/null @@ -1,28 +0,0 @@ -UnknownValueBinaryOpException -############################# - - Module-level binary operation on non-strict value 'lvalue [op] rvalue' is prohibited. - -This error indicates that you are attempting to perform a binary operation on -a value that can't be analyzed by strict modules (e.g. because it is imported -from a non-strict module). Because binary operations can be overridden in -Python to execute arbitrary code, doing this on an unknown value can cause -arbitrary side effects and is prohibited at top level of a strict module. - -.. code-block:: python - - from nonstrict import SOME_CONST - - MY_CONST = SOME_CONST + 1 - -This code will result in an error message such as: - -``UnknownValueBinaryOpException`` ' + 1' - -The error tells you strict modules' understanding of the values on each side -of the binary operation, and what the operation itself is. - -Typically the best solution to this situation is to make the module -containing ``SOME_CONST`` strict. - -You can look at :ref:`conversion_tips` for more ways to fix this error. diff --git a/cinderx/Docs/StrictModules/guide/errors/unknown_value_bool_op.rst b/cinderx/Docs/StrictModules/guide/errors/unknown_value_bool_op.rst deleted file mode 100644 index 52360a160..000000000 --- a/cinderx/Docs/StrictModules/guide/errors/unknown_value_bool_op.rst +++ /dev/null @@ -1,30 +0,0 @@ -UnknownValueBoolException -######################### - - Module-level conversion to bool on non-strict value 'value' is prohibited. - -This error indicates that you are attempting to convert a value to a bool, -either explicitly by calling bool(value) on it or implicitly by using it -in a conditional location (e.g. if, while, if expression, or, and, etc...) - - -.. code-block:: python - - from nonstrict_module import x - - if x: - pass - -This code will result in an error message such as: - -``UnknownValueBoolException`` 'Conversion to bool on unknown -value '{x imported from nonstrict_module}' is prohibited.' - -Here you can see the error tells you the value which is being checked for -truthiness. - -One possible solution to this is making the nonstrict module strict so that -x can be used at the top-level. If the non-strict module is something from -the Python standard module you might want to report a bug! - -You can look at :ref:`conversion_tips` for more ways to fix this error. diff --git a/cinderx/Docs/StrictModules/guide/errors/unknown_value_index.rst b/cinderx/Docs/StrictModules/guide/errors/unknown_value_index.rst deleted file mode 100644 index 7474cfab7..000000000 --- a/cinderx/Docs/StrictModules/guide/errors/unknown_value_index.rst +++ /dev/null @@ -1,28 +0,0 @@ -UnknownValueIndexException -########################## - - Module-level index into non-strict value 'value[index]' is prohibited. - -This error indicates that you are attempting to index into a value that -strict modules can't analyze. - -.. code-block:: python - - from nonstrict import GenericClass - - class MyClass(GenericClass[int]): - pass - -This code will result in an error message such as: - -``UnknownValueIndexException`` '[]' - -Here you can see the error tells you both what value we are accessing which -is not statically analyzable, but also what value we are using to -index into the unknown value (in this case the int type). - -One possible solution to this is making the nonstrict module strict so that -GenericClass can be used at the top-level. If the non-strict module is -something from the Python standard library you might want to report a bug! - -You can look at :ref:`conversion_tips` for more ways to fix this error. diff --git a/cinderx/Docs/StrictModules/guide/errors/unsafe_call.rst b/cinderx/Docs/StrictModules/guide/errors/unsafe_call.rst deleted file mode 100644 index e63a2af79..000000000 --- a/cinderx/Docs/StrictModules/guide/errors/unsafe_call.rst +++ /dev/null @@ -1,37 +0,0 @@ -UnsafeCallException -################### - - Call 'function()' may have side effects and is prohibited at module level. - -This error indicates that you are attempting to call a function whose execution -will cause side effects or has elements which can not be successfully verified. - - -.. code-block:: python - - import __strict__ - - def side_effects(): - print("I have side effects") - return 42 - - FORTY_TWO = side_effects() - -This code will result in an error message such as: - -``UnsafeCallException`` 'Call 'side_effects()' may have side effects and -is prohibited at module level.' - -In addition to this error from the call site you'll see an error about the -underlying violation in the function. Here the underlying error turned out -to be a :doc:`unknown_call`. In this case strict modules has no -definition of print because its sole purpose is to cause side effects: - -``UnknownValueCallException`` Call of unknown value 'print()' is prohibited. - -You can consider removing the prohibited operation from the function, -or move the call to the side effecting function out of the top-level. -You can look at :ref:`conversion_tips` for more ways to fix this error. - -If the underlying operation in the function is something you think strict -modules should support the analysis of you might want to report a bug! diff --git a/cinderx/Docs/StrictModules/guide/index.rst b/cinderx/Docs/StrictModules/guide/index.rst deleted file mode 100644 index ecde5a73b..000000000 --- a/cinderx/Docs/StrictModules/guide/index.rst +++ /dev/null @@ -1,75 +0,0 @@ -Users Guide for 3.10 -#################### - -Getting Started ---------------- - -Writing your first strict module is super easy. Check out the -:doc:`quickstart` to find out more. - -What does it mean if my module is strict? ------------------------------------------ -The short version of the strict-module rules: your module may not do anything -dynamic at import time that might have external side effects; it should just -define a set of names (constants, functions, and classes). - -While that might seem a little bit limiting, there is still a wealth of -built-ins that are supported. You can still do a lot of reflection over -types, use normal constructs like lists and dictionaries, comprehensions, -and a large number of built-in functions within your module definition. - -Making your modules strict means you receive benefits of strictness, including -but not limited to increased reliability and performance improvements. - - -Limitations ------------ -Strict modules undergo a number of different checks to ensure that they will -reliably and consistently produce the same module for the same input source -code. This ensures that top-level modules will always reliably succeed and -won't take dependencies on external environmental factors. - -.. toctree:: - :maxdepth: 1 - :titlesonly: - :glob: - - limitations/* - - -.. _conversion_tips: - -Conversion Tips ---------------- -This section includes tips on how to convert a non-strict module into a strict -module, common problems which might come up, and how you can effectively -:doc:`test ` your strict modules even though they're -immutable. - - -.. toctree:: - :maxdepth: 1 - :titlesonly: - :glob: - - conversion/* - - -Strict Module Errors / Exceptions ---------------------------------- -When converting a module to strict, or when modifying an existing strict -module, you may see many different errors reported by the linter. These will -have an exception name associated with them along with a detailed message -explaining what and where something went wrong in verifying a module as -strict. This section has detailed description of what error means and how you -can possibly fix it. - - - -.. toctree:: - :maxdepth: 1 - :titlesonly: - :glob: - - errors/index - errors/* diff --git a/cinderx/Docs/StrictModules/guide/limitations/builtins.rst b/cinderx/Docs/StrictModules/guide/limitations/builtins.rst deleted file mode 100644 index 8360044c3..000000000 --- a/cinderx/Docs/StrictModules/guide/limitations/builtins.rst +++ /dev/null @@ -1,56 +0,0 @@ -Supported Builtins -################## - -Strict modules support static verification for a subset of the standard Python -builtin types and functions. Usage of builtins not in this -list at the top-level of a module will result in an error message -and will prevent the module from being able to be marked as strict. - -If you need support for an additional built-in function or type outside of the -supported list please report a bug! - -Supported types and values: - -* AttributeError -* bool -* bytes -* classmethod -* complex -* dict -* object -* Ellipsis -* Exception -* float -* int -* list -* None -* NotImplemented -* property -* range -* set -* staticmethod -* str -* super -* type -* TypeError -* tuple -* ValueError - -Supported functions: - -* callable -* chr -* getattr -* hasattr -* len -* max -* min -* ord -* print -* isinstance -* setattr - -Unsupported functions: - -Currently exec and eval are disallowed at the top-level. In the future we may -allow their usage if they are used with deterministic strings. diff --git a/cinderx/Docs/StrictModules/guide/limitations/class_attrs.rst b/cinderx/Docs/StrictModules/guide/limitations/class_attrs.rst deleted file mode 100644 index 2c5c5bbe4..000000000 --- a/cinderx/Docs/StrictModules/guide/limitations/class_attrs.rst +++ /dev/null @@ -1,111 +0,0 @@ -Class Attributes -################ - -Strict modules transform the way attributes are handled in Python from being -diffused between types and instances to be entirely declared on the type. - -For normal Python definitions when you define a class, the class will have a -dictionary, its sub-types will have dictionaries, and the instances will also -have dictionaries. When an attribute is looked up you typically have to look in -at least the class dictionary and the instance dictionary. In strict modules -we've removed the instance dictionary and replaced it with attributes that -are always defined at the class level. We've done this by leveraging a -standard Python feature - ``__slots__``. - -There are a few different benefits from this transformation. The first is that -slots generally provide faster access to an attribute than instance -dictionaries - to access an instance field first the type's dictionary needs -to be checked, and then the instance dictionary can be checked. With this -transformation in place we only ever need to look at the type's dictionary -and we're done. We've also applied additional optimizations within Cinder to -further improve the performance of this layout. - -Another benefit is that it uses less memory - with a fixed number of slots -Python knows exactly the size of the instance that it needs to allocate to -store all of its fields. With a dictionary it may quickly need to be resized -multiple times while allocating the object, and dictionaries have a load factor -where a percentage of slots are necessarily unused. The dictionary is also its -own object instead of storing the attributes directly on the instance. - -And a final benefit is developer productivity. When you can arbitrarily attach -any attribute to an instance it's easy to make a mistake where you have a typo -on a member name and don't understand why it's not being updated. By not -allowing arbitrary fields to be assigned we turn this into an immediate and -obvious error. - -Class Attributes in Detail --------------------------- - -Now let's look look at some detailed examples. This first example shows what -you can do today in Python without a strict module to get the same benefits: - -.. code-block:: python - - class C: - __slots__ = ('myattr', ) - def __init__(self): - self.myattr = None - - a = C() - a.my_attr = 42 # AttributeError: 'C' object has no attribute 'my_attr' - -In this case we've defined a class using Python's ``__slots__`` feature and -specified that the type has an attribute "myattr". Later we've assigned -to "my_attr" accidentally. Without the presence of ``__slots__`` the Python -runtime would have happily allowed this assignment and we might have spent -a lot of time debugging while myattr doesn't have the right value. - -Strict modules handles this transformation to ``__slots__`` automatically and -will typically not require extra intervention on the behalf of the -programmer. If we put this code into a strict module all we have to do -is remove the ``__slots__`` entry and we get the exact same behavior: - -.. code-block:: python - - import __strict__ - - class C: - def __init__(self): - self.myattr = None - - a = C() - a.my_attr = 42 # AttributeError: 'C' object has no attribute 'my_attr' - -Strict modules will automatically populate the entries for ``__slots__`` based -upon the fields that are assigned in ``__init__``. If you have fields which you -don't want to eagerly populate you can also use Python's class level -annotations to indicate the presence of a field: - -.. code-block:: python - - class C: - myattr: int - - a = C() - a.myattr = 42 # OK - -We anticipate this shouldn't be much of an additional burden on developers -because these annotations are already used for providing typing information -to static analysis tools like Pyre. - -But these changes do have some subtle impacts - for example this code is -now an error where it wasn't before: - -.. code-block:: python - - class C: - def __init__(self): - self.f = 42 - - def f(self): - return 42 - - # Strict Module error: Class member conflicts with instance member: f - -The problem here is that there's now contention for storing two things in the -type - one is the method for "f", and the other is storing a descriptor (an -object which knows where to get or set the value in the instance) for the -instance attribute. It's not very often that users want to override a class -attribute with an instance one, but when it occurs you'll need to -resort to other techniques. For information on how to handle this see -:doc:`../conversion/class_inst_conflict`. diff --git a/cinderx/Docs/StrictModules/guide/limitations/deterministic.rst b/cinderx/Docs/StrictModules/guide/limitations/deterministic.rst deleted file mode 100644 index ec1b511a7..000000000 --- a/cinderx/Docs/StrictModules/guide/limitations/deterministic.rst +++ /dev/null @@ -1,27 +0,0 @@ -Deterministic Execution -####################### - -One of the most significant limitations, and the primary point of strict -modules, is to enforce that their contents are deterministic. This is -achieved by having an allow-list approach of what can occur inside of a strict -module at import time. The allow-list is composed of standard Python syntax -and the set of :doc:`builtins` which are allowed. Use of anything that -isn't analyzable will result in an error when importing a strict module. - -Strict modules themselves are verified to conform to the allow list by -an analysis done with an interpreter. The interpreter -will precisely simulate the execution of your code, analyzing loops, -flow control, exceptions and all other typical Python language elements -that are available. This means that you have available to you the full -breadth of the language. - -Strict modules will only validate code at the module -top-level - that includes elements such as top-level class declarations, -annotations on functions (unless `from __future__ import annotations` is -applied), etc. It will also analyze the result of calling any functions from -the top-level (which includes decorators on top-level definitions). - -The result is that while you are more limited in what you can -do within your module definitions, your actual functions aren't limited in -what they are allowed to do. Effectively a slightly more limited Python is -your meta-programming language for defining your Python programs. diff --git a/cinderx/Docs/StrictModules/guide/limitations/imports.rst b/cinderx/Docs/StrictModules/guide/limitations/imports.rst deleted file mode 100644 index 112eb8eea..000000000 --- a/cinderx/Docs/StrictModules/guide/limitations/imports.rst +++ /dev/null @@ -1,86 +0,0 @@ -Imports -####### - -Import Mechanics -================ - -There is also some impact on how imports behave. Because strict modules are -immutable, child packages cannot be set on them when the child package is -imported. Given that this is also a common source of errors and can cause -issues with order of imports this is a good thing. - -You might have some code which does a `from package import child` in one -spot, and elsewhere you might do `import package` and then try and access -`package.child`. When `package` is a strict module this will fail because -`child` is not published. - -Explicitly Imported Child Packages ----------------------------------- - -If you'd like to enable this programming model you can still explicitly -publish the child module on the parent package - -**package/__init__.py** - -.. code-block:: python - - import __strict__ - - from package import child - - -**package/child.py** - -.. code-block:: python - - import __strict__ - - def foo(): pass - -This pattern is also okay and the child package will be published on the -parent package. - - -Using Imports -============= - -You also may need to be a little bit careful about how imported values are -used within a strict module. In order to verify that a strict module has no -side effects you cannot interact with any values from non-strict modules at -the top-level of your module. While it may typically be obvious when you -are interacting with non-strict values there's at least one case when its -less obvious. - - -**package/__init__.py** - -.. code-block:: python - - - # I'm not strict - -**package/a.py** - -.. code-block:: python - - import __strict__ - - class C: - pass - -**strict_mod.py** - -.. code-block:: python - - import __strict__ - - from package import a - - class C(a.C): # not safe - pass - -In this case using "a" at the top-level is not safe because the package -itself isn't strict. Because the package isn't strict random code -could sneak in and replace "a" with a value which isn't the strict module. - -This can easily be solved by marking the package as strict. diff --git a/cinderx/Docs/StrictModules/guide/limitations/index.rst b/cinderx/Docs/StrictModules/guide/limitations/index.rst deleted file mode 100644 index fe4020424..000000000 --- a/cinderx/Docs/StrictModules/guide/limitations/index.rst +++ /dev/null @@ -1,14 +0,0 @@ -Limitations -########### - -Strict modules undergo a number of different checks to ensure that they will -reliably and consistently produce the same module for the same input source -code. This ensures that top-level modules will always reliably succeed and won’t -take dependencies on external environmental factors. - -.. toctree:: - :maxdepth: 1 - :titlesonly: - :glob: - - * diff --git a/cinderx/Docs/StrictModules/guide/limitations/ownership.rst b/cinderx/Docs/StrictModules/guide/limitations/ownership.rst deleted file mode 100644 index 58c673cd2..000000000 --- a/cinderx/Docs/StrictModules/guide/limitations/ownership.rst +++ /dev/null @@ -1,96 +0,0 @@ -Ownership -######### - -Strict modules are analyzed with a concept of ownership. That is, every value -that is produced from strict modules is owned by one and only one strict -module, and only that strict module is capable of mutating it. This -requirement ensures that modules are deterministic, and there isn't state is -the system which develops in an ad-hoc manner based upon how different modules -are imported. - -For a value to be owned by a module it doesn't need to actually be created -directly within the module which owns it. Rather the owning module is the -caller that ultimately causes the value to be created. Consider for example -this code: - -**a.py** - -.. code-block:: python - - import __strict__ - - def f(): - return {} - -**b.py** - -.. code-block:: python - - import __strict__ - - from a import f - - - x = f() - x["name"] = "value" - -This example is fine and will be permitted by strict modules; the owner of -the dictionary referred to by the variable `x` is module `b`. - -There are other useful patterns of this sort of modification. For example a -decorator can safely be applied in a module which will mutate the defining -function: - -**methods.py** - -.. code-block:: python - - import __strict__ - - def POST(f): - f.method = 'POST' - return f - -**routes.py** - -.. code-block:: python - - import __strict__ - - from methods import POST - - @POST - def create_user(*args): - ... - - -But it bans other patterns which you may be used to. For example you cannot -use a decorator to create a registry of functions: - -**methods.py** - -.. code-block:: python - - import __strict__ - - ROUTES = set() - - def route(f): - ROUTES.add(f) - return f - -**routes.py** - -.. code-block:: python - - import __strict__ - - from methods import route - - @route - def create_user(*args): - ... - - -This will result in a ``StrictModuleModifyImportedValueException`` " -from module methods is modified by routes; this is prohibited." diff --git a/cinderx/Docs/StrictModules/guide/quickstart.rst b/cinderx/Docs/StrictModules/guide/quickstart.rst deleted file mode 100644 index b063cbeaa..000000000 --- a/cinderx/Docs/StrictModules/guide/quickstart.rst +++ /dev/null @@ -1,70 +0,0 @@ -# Quickstart - -## How do I use it? - -Using Strict Modules requires a module loader able to detect strict modules -based on some marker (we use the presence of `import __strict__`). -Such a loader is included (at `compiler.strict.loader.StrictSourceFileLoader`) -and you can install it by calling `compiler.strict.loader.install()` in the -"main" module of your program (before anything else is imported.) -Note this means the main module itself cannot be strict. Alternatively, set the -`PYTHONINSTALLSTRICTLOADER` environment variable to a nonzero value, and -the loader will be installed for you (but then you can't customize the loader). - -## How do I make my module strict? - -To opt your module in, place the line ``import __strict__`` at the top of the -module. The ``__strict__`` marker line should come after the docstring if -present, after any ``from __future__ import`` statements, and before any -other imports. Comments can also precede the ``__strict__`` marker. - -If your module is marked as strict but violates the strict-mode rules, you -will get detailed errors when you try to import the module. - -> Note: The "launcher" module (the `__main__` in Python terms) cannot be marked -> strict, because by default, it must have one side-effect (of launching the -> application). - -What are the risks? -------------------- - -Most of the strict-mode restrictions have purely local effect; if you are -able to import your module after marking it strict, you're mostly good to go! -There are a couple runtime changes that can impact code outside the module: - -1. Strict mode makes the module itself and any classes in it immutable after -the module is done executing. This is most likely to impact tests that -monkeypatch the module or its classes. Refer to the :doc:`conversion/testing` -section to learn how to enable patching of strict moduels for testing. - -2. Instances of strict classes have `__slots__ -`_ automatically -created for them. This means they will raise ``AttributeError`` if you try to -add any attribute to them that isn't declared with a type annotation on the -class itself (e.g. ``attrname: int``) or assigned in the ``__init__`` method. -If you aren't confident that this isn't happening to your class somewhere in -the codebase, you can temporarily place the ``strict_modules.loose_slots`` -decorator on the class for a safer transition. See -:doc:`conversion/loose_slots` for details. - -What are the benefits? ----------------------- - -When you convert your module to strict, you immediately get these benefits: - -1. It becomes impossible to accidentally introduce import side effects in -your module, which prevents problems that can eat up debugging time or even -break prod. - -2. It becomes impossible to accidentally modify global state by mutating your -module or one of the classes in your module, also preventing bugs and test -flakiness. - -In the future, we hope that you will get other benefits too, like faster -imports when the module is unchanged since last import and production -efficiency improvements as well. - -What if I get a StrictModuleException? --------------------------------------- - -See :doc:`errors/index` for advice on handling errors in your strict module. diff --git a/cinderx/Immortalize/immortalize.cpp b/cinderx/Immortalize/immortalize.cpp index c9c14bd2a..b8069a090 100644 --- a/cinderx/Immortalize/immortalize.cpp +++ b/cinderx/Immortalize/immortalize.cpp @@ -8,14 +8,57 @@ #include "cinderx/Common/util.h" #include "cinderx/UpstreamBorrow/borrowed.h" // @donotremove +namespace cinderx { + +namespace { + #define FROM_GC(g) ((PyObject*)(((PyGC_Head*)g) + 1)) #define GEN_HEAD(state, n) (&(state)->generations[n].head) -using GCState = struct _gc_runtime_state; struct _gc_runtime_state* get_gc_state() { return &PyInterpreterState_Get()->gc; } +void immortalize_exact_dict_entries(PyObject* obj) { + PyObject* key; + PyObject* value; + Py_ssize_t pos = 0; + // PyDict_Next() can resolve lazy imports when values are requested. When + // lazy imports are enabled, heap immortalization must preserve them during + // prepare-for-fork. +#ifdef ENABLE_LAZY_IMPORTS + while (_PyDict_NextKeepLazy(obj, &pos, &key, &value)) { +#else + while (PyDict_Next(obj, &pos, &key, &value)) { +#endif + immortalize(key); + immortalize(value); + } +} + +// Code objects are not GC-traversed in 3.12, so their tuple fields can be +// immortal leaves with mortal entries. Keep this scoped to direct code tuple +// entries rather than changing global tuple semantics. +void immortalize_code_tuple_field(BorrowedRef<> obj) { + PyObject* tuple = obj.get(); + if (tuple == nullptr) { + return; + } + + immortalize(tuple); + + if (!PyTuple_CheckExact(tuple)) { + return; + } + + Py_ssize_t size = PyTuple_GET_SIZE(tuple); + for (Py_ssize_t i = 0; i < size; i++) { + immortalize(PyTuple_GET_ITEM(tuple, i)); + } +} + +} // namespace + bool can_immortalize(PyObject* obj) { if (obj == nullptr || _Py_IsImmortal(obj)) { return false; @@ -24,7 +67,7 @@ bool can_immortalize(PyObject* obj) { // Python 3.12 will assert that strings that are immortalized are also // interned in debug builds. This is purely a debug check, it's fine to do in // optimized builds. - if constexpr (PY_VERSION_HEX >= 0x030C0000 && kPyDebug) { + if constexpr (kPyDebug) { return !PyUnicode_Check(obj); } @@ -38,27 +81,25 @@ bool immortalize(PyObject* obj) { IMMORTALIZE(obj); + if (PyDict_CheckExact(obj)) { + immortalize_exact_dict_entries(obj); + } + if (PyCode_Check(obj)) { BorrowedRef code{obj}; codeExtra(code); -#if PY_VERSION_HEX < 0x030B0000 - // In 3.11 these changed to have the bytes embedded in the code object and - // the names in a unified tuple - IMMORTALIZE(PyCode_GetCode(code)); - IMMORTALIZE(PyCode_GetVarnames(code)); - IMMORTALIZE(PyCode_GetFreevars(code)); - IMMORTALIZE(PyCode_GetCellvars(code)); -#else IMMORTALIZE(code->co_localspluskinds); - IMMORTALIZE(code->co_localsplusnames); -#endif - IMMORTALIZE(code->co_consts); - IMMORTALIZE(code->co_names); IMMORTALIZE(code->co_linetable); + IMMORTALIZE(code->co_exceptiontable); + + immortalize_code_tuple_field(code->co_localsplusnames); + immortalize_code_tuple_field(code->co_consts); + immortalize_code_tuple_field(code->co_names); // These are strings and we need to check if this is safe. immortalize(code->co_filename); immortalize(code->co_name); + immortalize(code->co_qualname); } /* Cache the hash value of unicode object to reduce Copy-on-writes */ @@ -73,19 +114,28 @@ bool immortalize(PyObject* obj) { return true; } -#if PY_VERSION_HEX >= 0x030C0000 PyObject* immortalize_heap([[maybe_unused]] PyObject* mod) { -#ifdef Py_GIL_DISABLED - PyErr_SetString( - PyExc_RuntimeError, - "Immortalizing the heap is not yet supported in FT Python"); -#else + if constexpr (kFreeThreadedBuild) { + PyErr_SetString( + PyExc_RuntimeError, + "Immortalizing the heap is not yet supported in FT Python"); + return nullptr; + } + // TODO(T251571267): Low priority for now. /* Remove any dead objects to avoid immortalizing them */ PyGC_Collect(); /* Move all instances into the permanent generation */ - Cix_gc_freeze_impl(mod); + Ref<> gc_mod = Ref<>::steal(PyImport_ImportModule("gc")); + if (!gc_mod) { + return nullptr; + } + Ref<> freeze_result = + Ref<>::steal(PyObject_CallMethod(gc_mod, "freeze", nullptr)); + if (!freeze_result) { + return nullptr; + } /* Immortalize all instances in the permanent generation */ struct _gc_runtime_state* gcstate = get_gc_state(); @@ -101,24 +151,8 @@ PyObject* immortalize_heap([[maybe_unused]] PyObject* mod) { Py_TYPE(FROM_GC(gc)) ->tp_traverse(FROM_GC(gc), immortalize_visitor, nullptr); } -#endif Py_RETURN_NONE; } -#else -PyObject* immortalize_heap(PyObject* /* mod */) { - // for 3.10.cinder, we fall back to the implementation that ships in the gc - // module NOTE: this isn't a documented API, so I'm mostly adding it for - // parity, but it shouldn't actually be used anywhere - Ref<> gcmodule = Ref<>::steal(PyImport_ImportModule("gc")); - if (!gcmodule) { - return nullptr; - } - Ref<> immortalize = - Ref<>::steal(PyObject_GetAttrString(gcmodule, "immortalize_heap")); - if (!immortalize) { - return nullptr; - } - return Ref<>::steal(PyObject_CallFunctionObjArgs(immortalize, nullptr)); -} -#endif + +} // namespace cinderx diff --git a/cinderx/Immortalize/immortalize.h b/cinderx/Immortalize/immortalize.h index f94eca9d2..e23e1115f 100644 --- a/cinderx/Immortalize/immortalize.h +++ b/cinderx/Immortalize/immortalize.h @@ -6,9 +6,7 @@ #include -#if PY_VERSION_HEX < 0x030C0000 -#define _Py_IMMORTAL_REFCNT kImmortalInitialCount -#endif +namespace cinderx { /* * Immortalizes a Python object but does not check if that makes sense to do so. @@ -18,7 +16,7 @@ #define IMMORTALIZE(OBJ) Py_SET_IMMORTAL(OBJ) #elif PY_VERSION_HEX >= 0x030E0000 #define IMMORTALIZE(OBJ) Py_SET_REFCNT((OBJ), _Py_IMMORTAL_INITIAL_REFCNT) -#elif PY_VERSION_HEX >= 0x030C0000 +#else #define IMMORTALIZE(OBJ) Py_SET_REFCNT((OBJ), _Py_IMMORTAL_REFCNT) #endif @@ -35,7 +33,7 @@ bool immortalize(PyObject* obj); /* * Immortalize the Python objects currently on the heap. - * - * NOTE: In 3.10.cinder, this imports `gc` and calls `gc.immortalize_heap()` */ PyObject* immortalize_heap(PyObject* mod); + +} // namespace cinderx diff --git a/cinderx/Interpreter/3.10/cinder_opcode.h b/cinderx/Interpreter/3.10/cinder_opcode.h deleted file mode 100644 index a22c5f954..000000000 --- a/cinderx/Interpreter/3.10/cinder_opcode.h +++ /dev/null @@ -1,4 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. - -#include "cinderx/Interpreter/cinder_opcode_ids.h" -#include "cinderx/Interpreter/cinder_opcode_metadata.h" diff --git a/cinderx/Interpreter/3.10/cinder_opcode_ids.h b/cinderx/Interpreter/3.10/cinder_opcode_ids.h deleted file mode 100644 index 009b78213..000000000 --- a/cinderx/Interpreter/3.10/cinder_opcode_ids.h +++ /dev/null @@ -1,269 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. - -/* Auto-generated by Tools/scripts/generate_opcode_h.py from Lib/opcode.py */ -#ifndef Py_OPCODE_H -#define Py_OPCODE_H -#ifdef __cplusplus -extern "C" { -#endif - - - /* Instruction opcodes for compiled code */ -#define PY_OPCODES(X) \ - X(POP_TOP, 1) \ - X(ROT_TWO, 2) \ - X(ROT_THREE, 3) \ - X(DUP_TOP, 4) \ - X(DUP_TOP_TWO, 5) \ - X(ROT_FOUR, 6) \ - X(NOP, 9) \ - X(UNARY_POSITIVE, 10) \ - X(UNARY_NEGATIVE, 11) \ - X(UNARY_NOT, 12) \ - X(UNARY_INVERT, 15) \ - X(BINARY_MATRIX_MULTIPLY, 16) \ - X(INPLACE_MATRIX_MULTIPLY, 17) \ - X(BINARY_POWER, 19) \ - X(BINARY_MULTIPLY, 20) \ - X(BINARY_MODULO, 22) \ - X(BINARY_ADD, 23) \ - X(BINARY_SUBTRACT, 24) \ - X(BINARY_SUBSCR, 25) \ - X(BINARY_FLOOR_DIVIDE, 26) \ - X(BINARY_TRUE_DIVIDE, 27) \ - X(INPLACE_FLOOR_DIVIDE, 28) \ - X(INPLACE_TRUE_DIVIDE, 29) \ - X(GET_LEN, 30) \ - X(MATCH_MAPPING, 31) \ - X(MATCH_SEQUENCE, 32) \ - X(MATCH_KEYS, 33) \ - X(COPY_DICT_WITHOUT_KEYS, 34) \ - X(WITH_EXCEPT_START, 49) \ - X(GET_AITER, 50) \ - X(GET_ANEXT, 51) \ - X(BEFORE_ASYNC_WITH, 52) \ - X(END_ASYNC_FOR, 54) \ - X(INPLACE_ADD, 55) \ - X(INPLACE_SUBTRACT, 56) \ - X(INPLACE_MULTIPLY, 57) \ - X(INPLACE_MODULO, 59) \ - X(STORE_SUBSCR, 60) \ - X(DELETE_SUBSCR, 61) \ - X(BINARY_LSHIFT, 62) \ - X(BINARY_RSHIFT, 63) \ - X(BINARY_AND, 64) \ - X(BINARY_XOR, 65) \ - X(BINARY_OR, 66) \ - X(INPLACE_POWER, 67) \ - X(GET_ITER, 68) \ - X(GET_YIELD_FROM_ITER, 69) \ - X(PRINT_EXPR, 70) \ - X(LOAD_BUILD_CLASS, 71) \ - X(YIELD_FROM, 72) \ - X(GET_AWAITABLE, 73) \ - X(LOAD_ASSERTION_ERROR, 74) \ - X(INPLACE_LSHIFT, 75) \ - X(INPLACE_RSHIFT, 76) \ - X(INPLACE_AND, 77) \ - X(INPLACE_XOR, 78) \ - X(INPLACE_OR, 79) \ - X(LIST_TO_TUPLE, 82) \ - X(RETURN_VALUE, 83) \ - X(IMPORT_STAR, 84) \ - X(SETUP_ANNOTATIONS, 85) \ - X(YIELD_VALUE, 86) \ - X(POP_BLOCK, 87) \ - X(POP_EXCEPT, 89) \ - X(HAVE_ARGUMENT, 90) \ - X(STORE_NAME, 90) \ - X(DELETE_NAME, 91) \ - X(UNPACK_SEQUENCE, 92) \ - X(FOR_ITER, 93) \ - X(UNPACK_EX, 94) \ - X(STORE_ATTR, 95) \ - X(DELETE_ATTR, 96) \ - X(STORE_GLOBAL, 97) \ - X(DELETE_GLOBAL, 98) \ - X(ROT_N, 99) \ - X(LOAD_CONST, 100) \ - X(LOAD_NAME, 101) \ - X(BUILD_TUPLE, 102) \ - X(BUILD_LIST, 103) \ - X(BUILD_SET, 104) \ - X(BUILD_MAP, 105) \ - X(LOAD_ATTR, 106) \ - X(COMPARE_OP, 107) \ - X(IMPORT_NAME, 108) \ - X(IMPORT_FROM, 109) \ - X(JUMP_FORWARD, 110) \ - X(JUMP_IF_FALSE_OR_POP, 111) \ - X(JUMP_IF_TRUE_OR_POP, 112) \ - X(JUMP_ABSOLUTE, 113) \ - X(POP_JUMP_IF_FALSE, 114) \ - X(POP_JUMP_IF_TRUE, 115) \ - X(LOAD_GLOBAL, 116) \ - X(IS_OP, 117) \ - X(CONTAINS_OP, 118) \ - X(RERAISE, 119) \ - X(JUMP_IF_NOT_EXC_MATCH, 121) \ - X(SETUP_FINALLY, 122) \ - X(LOAD_FAST, 124) \ - X(STORE_FAST, 125) \ - X(DELETE_FAST, 126) \ - X(GEN_START, 129) \ - X(RAISE_VARARGS, 130) \ - X(CALL_FUNCTION, 131) \ - X(MAKE_FUNCTION, 132) \ - X(BUILD_SLICE, 133) \ - X(LOAD_CLOSURE, 135) \ - X(LOAD_DEREF, 136) \ - X(STORE_DEREF, 137) \ - X(DELETE_DEREF, 138) \ - X(CALL_FUNCTION_KW, 141) \ - X(CALL_FUNCTION_EX, 142) \ - X(SETUP_WITH, 143) \ - X(EXTENDED_ARG, 144) \ - X(LIST_APPEND, 145) \ - X(SET_ADD, 146) \ - X(MAP_ADD, 147) \ - X(LOAD_CLASSDEREF, 148) \ - X(MATCH_CLASS, 152) \ - X(SETUP_ASYNC_WITH, 154) \ - X(FORMAT_VALUE, 155) \ - X(BUILD_CONST_KEY_MAP, 156) \ - X(BUILD_STRING, 157) \ - X(INVOKE_METHOD, 158) \ - X(LOAD_FIELD, 159) \ - X(LOAD_METHOD, 160) \ - X(CALL_METHOD, 161) \ - X(LIST_EXTEND, 162) \ - X(SET_UPDATE, 163) \ - X(DICT_MERGE, 164) \ - X(DICT_UPDATE, 165) \ - X(STORE_FIELD, 166) \ - X(BUILD_CHECKED_LIST, 168) \ - X(LOAD_TYPE, 169) \ - X(CAST, 170) \ - X(LOAD_LOCAL, 171) \ - X(STORE_LOCAL, 172) \ - X(PRIMITIVE_BOX, 174) \ - X(POP_JUMP_IF_ZERO, 175) \ - X(POP_JUMP_IF_NONZERO, 176) \ - X(PRIMITIVE_UNBOX, 177) \ - X(PRIMITIVE_BINARY_OP, 178) \ - X(PRIMITIVE_UNARY_OP, 179) \ - X(PRIMITIVE_COMPARE_OP, 180) \ - X(LOAD_ITERABLE_ARG, 181) \ - X(LOAD_MAPPING_ARG, 182) \ - X(INVOKE_FUNCTION, 183) \ - X(JUMP_IF_ZERO_OR_POP, 184) \ - X(JUMP_IF_NONZERO_OR_POP, 185) \ - X(FAST_LEN, 186) \ - X(CONVERT_PRIMITIVE, 187) \ - X(INVOKE_NATIVE, 189) \ - X(LOAD_CLASS, 190) \ - X(BUILD_CHECKED_MAP, 191) \ - X(SEQUENCE_GET, 192) \ - X(SEQUENCE_SET, 193) \ - X(LIST_DEL, 194) \ - X(REFINE_TYPE, 195) \ - X(PRIMITIVE_LOAD_CONST, 196) \ - X(RETURN_PRIMITIVE, 197) \ - X(LOAD_METHOD_SUPER, 198) \ - X(LOAD_ATTR_SUPER, 199) \ - X(TP_ALLOC, 200) \ - X(LOAD_METHOD_STATIC, 203) \ - X(LOAD_METHOD_UNSHADOWED_METHOD, 205) \ - X(LOAD_METHOD_TYPE_METHODLIKE, 206) \ - X(BUILD_CHECKED_LIST_CACHED, 207) \ - X(TP_ALLOC_CACHED, 208) \ - X(LOAD_ATTR_S_MODULE, 209) \ - X(LOAD_METHOD_S_MODULE, 210) \ - X(INVOKE_FUNCTION_CACHED, 211) \ - X(INVOKE_FUNCTION_INDIRECT_CACHED, 212) \ - X(BUILD_CHECKED_MAP_CACHED, 213) \ - X(LOAD_METHOD_STATIC_CACHED, 214) \ - X(PRIMITIVE_STORE_FAST, 215) \ - X(CAST_CACHED_OPTIONAL, 216) \ - X(CAST_CACHED, 217) \ - X(CAST_CACHED_EXACT, 218) \ - X(CAST_CACHED_OPTIONAL_EXACT, 219) \ - X(LOAD_PRIMITIVE_FIELD, 220) \ - X(STORE_PRIMITIVE_FIELD, 221) \ - X(LOAD_OBJ_FIELD, 222) \ - X(STORE_OBJ_FIELD, 223) \ - X(INVOKE_METHOD_CACHED, 224) \ - X(BINARY_SUBSCR_TUPLE_CONST_INT, 225) \ - X(BINARY_SUBSCR_DICT_STR, 226) \ - X(BINARY_SUBSCR_LIST, 227) \ - X(BINARY_SUBSCR_TUPLE, 228) \ - X(BINARY_SUBSCR_DICT, 229) \ - X(LOAD_METHOD_UNCACHABLE, 230) \ - X(LOAD_METHOD_MODULE, 231) \ - X(LOAD_METHOD_TYPE, 232) \ - X(LOAD_METHOD_SPLIT_DICT_DESCR, 233) \ - X(LOAD_METHOD_SPLIT_DICT_METHOD, 234) \ - X(LOAD_METHOD_DICT_DESCR, 235) \ - X(LOAD_METHOD_DICT_METHOD, 236) \ - X(LOAD_METHOD_NO_DICT_METHOD, 237) \ - X(LOAD_METHOD_NO_DICT_DESCR, 238) \ - X(STORE_ATTR_SLOT, 239) \ - X(STORE_ATTR_SPLIT_DICT, 240) \ - X(STORE_ATTR_DESCR, 241) \ - X(STORE_ATTR_UNCACHABLE, 242) \ - X(STORE_ATTR_DICT, 243) \ - X(LOAD_ATTR_POLYMORPHIC, 244) \ - X(LOAD_ATTR_SLOT, 245) \ - X(LOAD_ATTR_MODULE, 246) \ - X(LOAD_ATTR_TYPE, 247) \ - X(LOAD_ATTR_SPLIT_DICT_DESCR, 248) \ - X(LOAD_ATTR_SPLIT_DICT, 249) \ - X(LOAD_ATTR_DICT_NO_DESCR, 250) \ - X(LOAD_ATTR_NO_DICT_DESCR, 251) \ - X(LOAD_ATTR_DICT_DESCR, 252) \ - X(LOAD_ATTR_UNCACHABLE, 253) \ - X(LOAD_GLOBAL_CACHED, 254) \ - X(SHADOW_NOP, 255) - -#ifdef NEED_OPCODE_JUMP_TABLES -static uint32_t _PyOpcode_RelativeJump[8] = { - 0U, - 0U, - 536870912U, - 67125248U, - 67141632U, - 0U, - 0U, - 0U, -}; -static uint32_t _PyOpcode_Jump[8] = { - 0U, - 0U, - 536870912U, - 101695488U, - 67141632U, - 50429952U, - 0U, - 0U, -}; -#endif /* OPCODE_TABLES */ - - -enum { -#define OP(op, value) op = value, -PY_OPCODES(OP) -#undef OP -}; - -/* EXCEPT_HANDLER is a special, implicit block type which is created when - entering an except handler. It is not an opcode but we define it here - as we want it to be available to both frameobject.c and ceval.c, while - remaining private.*/ -#define EXCEPT_HANDLER 257 - -#define HAS_ARG(op) ((op) >= HAVE_ARGUMENT) - -#ifdef __cplusplus -} -#endif -#endif /* !Py_OPCODE_H */ diff --git a/cinderx/Interpreter/3.10/cinder_opcode_metadata.h b/cinderx/Interpreter/3.10/cinder_opcode_metadata.h deleted file mode 100644 index dde8dfe20..000000000 --- a/cinderx/Interpreter/3.10/cinder_opcode_metadata.h +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. - -#include "cinderx/Interpreter/cinder_opcode_ids.h" - -#ifdef NEED_OPCODE_NAMES - -// Note: Some opcodes share the same number (e.g., HAVE_ARGUMENT and STORE_NAME -// are both 90). The designated initializer will use the last one. -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Winitializer-overrides" -#endif - -static const char *const _CiOpcode_OpName[256] = { -#define OPCODE_NAME(name, num) [num] = #name, - PY_OPCODES(OPCODE_NAME) -#undef OPCODE_NAME -}; - -#ifdef __clang__ -#pragma clang diagnostic pop -#endif - -#endif diff --git a/cinderx/Interpreter/3.10/cinderx_opcode_targets.h b/cinderx/Interpreter/3.10/cinderx_opcode_targets.h deleted file mode 100644 index 34fea411c..000000000 --- a/cinderx/Interpreter/3.10/cinderx_opcode_targets.h +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. - -static void *opcode_targets[256] = { - &&_unknown_opcode, - &&TARGET_POP_TOP, - &&TARGET_ROT_TWO, - &&TARGET_ROT_THREE, - &&TARGET_DUP_TOP, - &&TARGET_DUP_TOP_TWO, - &&TARGET_ROT_FOUR, - &&_unknown_opcode, - &&_unknown_opcode, - &&TARGET_NOP, - &&TARGET_UNARY_POSITIVE, - &&TARGET_UNARY_NEGATIVE, - &&TARGET_UNARY_NOT, - &&_unknown_opcode, - &&_unknown_opcode, - &&TARGET_UNARY_INVERT, - &&TARGET_BINARY_MATRIX_MULTIPLY, - &&TARGET_INPLACE_MATRIX_MULTIPLY, - &&_unknown_opcode, - &&TARGET_BINARY_POWER, - &&TARGET_BINARY_MULTIPLY, - &&_unknown_opcode, - &&TARGET_BINARY_MODULO, - &&TARGET_BINARY_ADD, - &&TARGET_BINARY_SUBTRACT, - &&TARGET_BINARY_SUBSCR, - &&TARGET_BINARY_FLOOR_DIVIDE, - &&TARGET_BINARY_TRUE_DIVIDE, - &&TARGET_INPLACE_FLOOR_DIVIDE, - &&TARGET_INPLACE_TRUE_DIVIDE, - &&TARGET_GET_LEN, - &&TARGET_MATCH_MAPPING, - &&TARGET_MATCH_SEQUENCE, - &&TARGET_MATCH_KEYS, - &&TARGET_COPY_DICT_WITHOUT_KEYS, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&TARGET_WITH_EXCEPT_START, - &&TARGET_GET_AITER, - &&TARGET_GET_ANEXT, - &&TARGET_BEFORE_ASYNC_WITH, - &&_unknown_opcode, - &&TARGET_END_ASYNC_FOR, - &&TARGET_INPLACE_ADD, - &&TARGET_INPLACE_SUBTRACT, - &&TARGET_INPLACE_MULTIPLY, - &&_unknown_opcode, - &&TARGET_INPLACE_MODULO, - &&TARGET_STORE_SUBSCR, - &&TARGET_DELETE_SUBSCR, - &&TARGET_BINARY_LSHIFT, - &&TARGET_BINARY_RSHIFT, - &&TARGET_BINARY_AND, - &&TARGET_BINARY_XOR, - &&TARGET_BINARY_OR, - &&TARGET_INPLACE_POWER, - &&TARGET_GET_ITER, - &&TARGET_GET_YIELD_FROM_ITER, - &&TARGET_PRINT_EXPR, - &&TARGET_LOAD_BUILD_CLASS, - &&TARGET_YIELD_FROM, - &&TARGET_GET_AWAITABLE, - &&TARGET_LOAD_ASSERTION_ERROR, - &&TARGET_INPLACE_LSHIFT, - &&TARGET_INPLACE_RSHIFT, - &&TARGET_INPLACE_AND, - &&TARGET_INPLACE_XOR, - &&TARGET_INPLACE_OR, - &&_unknown_opcode, - &&_unknown_opcode, - &&TARGET_LIST_TO_TUPLE, - &&TARGET_RETURN_VALUE, - &&TARGET_IMPORT_STAR, - &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_YIELD_VALUE, - &&TARGET_POP_BLOCK, - &&_unknown_opcode, - &&TARGET_POP_EXCEPT, - &&TARGET_STORE_NAME, - &&TARGET_DELETE_NAME, - &&TARGET_UNPACK_SEQUENCE, - &&TARGET_FOR_ITER, - &&TARGET_UNPACK_EX, - &&TARGET_STORE_ATTR, - &&TARGET_DELETE_ATTR, - &&TARGET_STORE_GLOBAL, - &&TARGET_DELETE_GLOBAL, - &&TARGET_ROT_N, - &&TARGET_LOAD_CONST, - &&TARGET_LOAD_NAME, - &&TARGET_BUILD_TUPLE, - &&TARGET_BUILD_LIST, - &&TARGET_BUILD_SET, - &&TARGET_BUILD_MAP, - &&TARGET_LOAD_ATTR, - &&TARGET_COMPARE_OP, - &&TARGET_IMPORT_NAME, - &&TARGET_IMPORT_FROM, - &&TARGET_JUMP_FORWARD, - &&TARGET_JUMP_IF_FALSE_OR_POP, - &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_JUMP_ABSOLUTE, - &&TARGET_POP_JUMP_IF_FALSE, - &&TARGET_POP_JUMP_IF_TRUE, - &&TARGET_LOAD_GLOBAL, - &&TARGET_IS_OP, - &&TARGET_CONTAINS_OP, - &&TARGET_RERAISE, - &&_unknown_opcode, - &&TARGET_JUMP_IF_NOT_EXC_MATCH, - &&TARGET_SETUP_FINALLY, - &&_unknown_opcode, - &&TARGET_LOAD_FAST, - &&TARGET_STORE_FAST, - &&TARGET_DELETE_FAST, - &&_unknown_opcode, - &&_unknown_opcode, - &&TARGET_GEN_START, - &&TARGET_RAISE_VARARGS, - &&TARGET_CALL_FUNCTION, - &&TARGET_MAKE_FUNCTION, - &&TARGET_BUILD_SLICE, - &&_unknown_opcode, - &&TARGET_LOAD_CLOSURE, - &&TARGET_LOAD_DEREF, - &&TARGET_STORE_DEREF, - &&TARGET_DELETE_DEREF, - &&_unknown_opcode, - &&_unknown_opcode, - &&TARGET_CALL_FUNCTION_KW, - &&TARGET_CALL_FUNCTION_EX, - &&TARGET_SETUP_WITH, - &&TARGET_EXTENDED_ARG, - &&TARGET_LIST_APPEND, - &&TARGET_SET_ADD, - &&TARGET_MAP_ADD, - &&TARGET_LOAD_CLASSDEREF, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&TARGET_MATCH_CLASS, - &&_unknown_opcode, - &&TARGET_SETUP_ASYNC_WITH, - &&TARGET_FORMAT_VALUE, - &&TARGET_BUILD_CONST_KEY_MAP, - &&TARGET_BUILD_STRING, - &&TARGET_INVOKE_METHOD, - &&TARGET_LOAD_FIELD, - &&TARGET_LOAD_METHOD, - &&TARGET_CALL_METHOD, - &&TARGET_LIST_EXTEND, - &&TARGET_SET_UPDATE, - &&TARGET_DICT_MERGE, - &&TARGET_DICT_UPDATE, - &&TARGET_STORE_FIELD, - &&_unknown_opcode, - &&TARGET_BUILD_CHECKED_LIST, - &&TARGET_LOAD_TYPE, - &&TARGET_CAST, - &&TARGET_LOAD_LOCAL, - &&TARGET_STORE_LOCAL, - &&_unknown_opcode, - &&TARGET_PRIMITIVE_BOX, - &&TARGET_POP_JUMP_IF_ZERO, - &&TARGET_POP_JUMP_IF_NONZERO, - &&TARGET_PRIMITIVE_UNBOX, - &&TARGET_PRIMITIVE_BINARY_OP, - &&TARGET_PRIMITIVE_UNARY_OP, - &&TARGET_PRIMITIVE_COMPARE_OP, - &&TARGET_LOAD_ITERABLE_ARG, - &&TARGET_LOAD_MAPPING_ARG, - &&TARGET_INVOKE_FUNCTION, - &&TARGET_JUMP_IF_ZERO_OR_POP, - &&TARGET_JUMP_IF_NONZERO_OR_POP, - &&TARGET_FAST_LEN, - &&TARGET_CONVERT_PRIMITIVE, - &&_unknown_opcode, - &&TARGET_INVOKE_NATIVE, - &&TARGET_LOAD_CLASS, - &&TARGET_BUILD_CHECKED_MAP, - &&TARGET_SEQUENCE_GET, - &&TARGET_SEQUENCE_SET, - &&TARGET_LIST_DEL, - &&TARGET_REFINE_TYPE, - &&TARGET_PRIMITIVE_LOAD_CONST, - &&TARGET_RETURN_PRIMITIVE, - &&TARGET_LOAD_METHOD_SUPER, - &&TARGET_LOAD_ATTR_SUPER, - &&TARGET_TP_ALLOC, - &&_unknown_opcode, - &&_unknown_opcode, - &&TARGET_LOAD_METHOD_STATIC, - &&_unknown_opcode, - &&TARGET_LOAD_METHOD_UNSHADOWED_METHOD, - &&TARGET_LOAD_METHOD_TYPE_METHODLIKE, - &&TARGET_BUILD_CHECKED_LIST_CACHED, - &&TARGET_TP_ALLOC_CACHED, - &&TARGET_LOAD_ATTR_S_MODULE, - &&TARGET_LOAD_METHOD_S_MODULE, - &&TARGET_INVOKE_FUNCTION_CACHED, - &&TARGET_INVOKE_FUNCTION_INDIRECT_CACHED, - &&TARGET_BUILD_CHECKED_MAP_CACHED, - &&TARGET_LOAD_METHOD_STATIC_CACHED, - &&TARGET_PRIMITIVE_STORE_FAST, - &&TARGET_CAST_CACHED_OPTIONAL, - &&TARGET_CAST_CACHED, - &&TARGET_CAST_CACHED_EXACT, - &&TARGET_CAST_CACHED_OPTIONAL_EXACT, - &&TARGET_LOAD_PRIMITIVE_FIELD, - &&TARGET_STORE_PRIMITIVE_FIELD, - &&TARGET_LOAD_OBJ_FIELD, - &&TARGET_STORE_OBJ_FIELD, - &&TARGET_INVOKE_METHOD_CACHED, - &&TARGET_BINARY_SUBSCR_TUPLE_CONST_INT, - &&TARGET_BINARY_SUBSCR_DICT_STR, - &&TARGET_BINARY_SUBSCR_LIST, - &&TARGET_BINARY_SUBSCR_TUPLE, - &&TARGET_BINARY_SUBSCR_DICT, - &&TARGET_LOAD_METHOD_UNCACHABLE, - &&TARGET_LOAD_METHOD_MODULE, - &&TARGET_LOAD_METHOD_TYPE, - &&TARGET_LOAD_METHOD_SPLIT_DICT_DESCR, - &&TARGET_LOAD_METHOD_SPLIT_DICT_METHOD, - &&TARGET_LOAD_METHOD_DICT_DESCR, - &&TARGET_LOAD_METHOD_DICT_METHOD, - &&TARGET_LOAD_METHOD_NO_DICT_METHOD, - &&TARGET_LOAD_METHOD_NO_DICT_DESCR, - &&TARGET_STORE_ATTR_SLOT, - &&TARGET_STORE_ATTR_SPLIT_DICT, - &&TARGET_STORE_ATTR_DESCR, - &&TARGET_STORE_ATTR_UNCACHABLE, - &&TARGET_STORE_ATTR_DICT, - &&TARGET_LOAD_ATTR_POLYMORPHIC, - &&TARGET_LOAD_ATTR_SLOT, - &&TARGET_LOAD_ATTR_MODULE, - &&TARGET_LOAD_ATTR_TYPE, - &&TARGET_LOAD_ATTR_SPLIT_DICT_DESCR, - &&TARGET_LOAD_ATTR_SPLIT_DICT, - &&TARGET_LOAD_ATTR_DICT_NO_DESCR, - &&TARGET_LOAD_ATTR_NO_DICT_DESCR, - &&TARGET_LOAD_ATTR_DICT_DESCR, - &&TARGET_LOAD_ATTR_UNCACHABLE, - &&TARGET_LOAD_GLOBAL_CACHED, - &&TARGET_SHADOW_NOP, - -}; diff --git a/cinderx/Interpreter/3.10/generate_opcode_h.py b/cinderx/Interpreter/3.10/generate_opcode_h.py deleted file mode 100644 index f75b43638..000000000 --- a/cinderx/Interpreter/3.10/generate_opcode_h.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. - -# This script generates the opcode.h header file. - -# Do not format this with black; it is forked from python/3.12, and we would -# like to keep it diff-friendly. Likewise, do not lint it. -# fmt: off -# flake8: noqa - -import functools -import sys -import tokenize - -header = """ -/* Auto-generated by Tools/scripts/generate_opcode_h.py from Lib/opcode.py */ -#ifndef Py_OPCODE_H -#define Py_OPCODE_H -#ifdef __cplusplus -extern "C" { -#endif - - - /* Instruction opcodes for compiled code */ -#define PY_OPCODES(X)""".lstrip() - -footer = """ - -enum { -#define OP(op, value) op = value, -PY_OPCODES(OP) -#undef OP -}; - -/* EXCEPT_HANDLER is a special, implicit block type which is created when - entering an except handler. It is not an opcode but we define it here - as we want it to be available to both frameobject.c and ceval.c, while - remaining private.*/ -#define EXCEPT_HANDLER 257 - -#define HAS_ARG(op) ((op) >= HAVE_ARGUMENT) - -#ifdef __cplusplus -} -#endif -#endif /* !Py_OPCODE_H */ -""" - -UINT32_MASK = (1<<32)-1 - -def write_int_array_from_ops(name, ops, out): - bits = 0 - for op in ops: - bits |= 1<>= 32 - assert bits == 0 - out.write(f"}};\n") - -def main(opcode_py, outfile='Include/opcode.h'): - opcode = {} - if hasattr(tokenize, 'open'): - fp = tokenize.open(opcode_py) # Python 3.2+ - else: - fp = open(opcode_py) # Python 2.7 - with fp: - code = fp.read() - exec(code, opcode) - opmap = opcode['opmap'] - hasjrel = opcode['hasjrel'] - hasjabs = opcode['hasjabs'] - - max_op_len = functools.reduce( - lambda m, elem: max(m, len(elem)), opcode['opname'], 0 - ) + 3 # 3-digit opcode length - - def write_line(opname, opnum): - padding = max_op_len - len(opname) - fobj.write(" \\\n X(%s, %*d)" % (opname, padding, opnum)) - - with open(outfile, 'w') as fobj: - fobj.write(header) - for name in opcode['opname']: - if name in opmap: - write_line(name, opmap[name]) - if name == 'POP_EXCEPT': # Special entry for HAVE_ARGUMENT - write_line('HAVE_ARGUMENT', opcode['HAVE_ARGUMENT']) - - fobj.write("\n\n#ifdef NEED_OPCODE_JUMP_TABLES\n") - write_int_array_from_ops("_PyOpcode_RelativeJump", opcode['hasjrel'], fobj) - write_int_array_from_ops("_PyOpcode_Jump", opcode['hasjrel'] + opcode['hasjabs'], fobj) - fobj.write("#endif /* OPCODE_TABLES */\n") - - fobj.write(footer) - - print("%s regenerated from %s" % (outfile, opcode_py)) - - -if __name__ == '__main__': - main(sys.argv[1], sys.argv[2]) diff --git a/cinderx/Interpreter/3.10/interpreter.c b/cinderx/Interpreter/3.10/interpreter.c deleted file mode 100644 index 503405e32..000000000 --- a/cinderx/Interpreter/3.10/interpreter.c +++ /dev/null @@ -1,5189 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. - -#include "cinderx/Interpreter/cinder_opcode.h" - -#define CINDERX_INTERPRETER -#ifdef FBCODE_BUILD -#include "ceval.c" -#else -#include "../../Python/ceval.c" -#endif - -#include "cinderx/Common/code.h" -#include "cinderx/Common/extra-py-flags.h" -#include "cinderx/Interpreter/iter_helpers.h" -#include "cinderx/Shadowcode/shadowcode.h" -#include "cinderx/StaticPython/checked_dict.h" -#include "cinderx/StaticPython/checked_list.h" -#include "cinderx/StaticPython/classloader.h" -#include "cinderx/StaticPython/errors.h" -#include "cinderx/StaticPython/static_array.h" -#include "cinderx/UpstreamBorrow/borrowed.h" - -// These are used to truncate primitives/check signed bits when converting -// between them -static uint64_t trunc_masks[] = {0xFF, 0xFFFF, 0xFFFFFFFF, 0xFFFFFFFFFFFFFFFF}; -static uint64_t signed_bits[] = {0x80, 0x8000, 0x80000000, 0x8000000000000000}; -static uint64_t signex_masks[] = { - 0xFFFFFFFFFFFFFF00, - 0xFFFFFFFFFFFF0000, - 0xFFFFFFFF00000000, - 0x0}; - -// #ifdef HAVE_ERRNO_H -// #include -// #endif -// #include "ceval_gil.h" - -static inline int8_t unbox_primitive_bool_and_decref(PyObject* x) { - assert(PyBool_Check(x)); - int8_t res = (x == Py_True) ? 1 : 0; - Py_DECREF(x); - return res; -} - -static inline Py_ssize_t unbox_primitive_int_and_decref(PyObject* x) { - assert(PyLong_Check(x)); - Py_ssize_t res = (Py_ssize_t)PyLong_AsVoidPtr(x); - Py_DECREF(x); - return res; -} - -static inline void store_field(int field_type, void* addr, PyObject* value) { - switch (field_type) { - case TYPED_BOOL: - *(int8_t*)addr = (int8_t)unbox_primitive_bool_and_decref(value); - break; - case TYPED_INT8: - *(int8_t*)addr = (int8_t)unbox_primitive_int_and_decref(value); - break; - case TYPED_INT16: - *(int16_t*)addr = (int16_t)unbox_primitive_int_and_decref(value); - break; - case TYPED_INT32: - *(int32_t*)addr = (int32_t)unbox_primitive_int_and_decref(value); - break; - case TYPED_INT64: - *(int64_t*)addr = (int64_t)unbox_primitive_int_and_decref(value); - break; - case TYPED_UINT8: - *(uint8_t*)addr = (uint8_t)unbox_primitive_int_and_decref(value); - break; - case TYPED_UINT16: - *(uint16_t*)addr = (uint16_t)unbox_primitive_int_and_decref(value); - break; - case TYPED_UINT32: - *(uint32_t*)addr = (uint32_t)unbox_primitive_int_and_decref(value); - break; - case TYPED_UINT64: - *(uint64_t*)addr = (uint64_t)unbox_primitive_int_and_decref(value); - break; - case TYPED_DOUBLE: - *((double*)addr) = PyFloat_AsDouble(value); - Py_DECREF(value); - break; - default: - PyErr_SetString(PyExc_RuntimeError, "unsupported field type"); - } -} - -static inline PyObject* load_field(int field_type, void* addr) { - PyObject* value; - switch (field_type) { - case TYPED_BOOL: - value = PyBool_FromLong(*(int8_t*)addr); - break; - case TYPED_INT8: - value = PyLong_FromVoidPtr((void*)(Py_ssize_t) * ((int8_t*)addr)); - break; - case TYPED_INT16: - value = PyLong_FromVoidPtr((void*)(Py_ssize_t) * ((int16_t*)addr)); - break; - case TYPED_INT32: - value = PyLong_FromVoidPtr((void*)(Py_ssize_t) * ((int32_t*)addr)); - break; - case TYPED_INT64: - value = PyLong_FromVoidPtr((void*)(Py_ssize_t) * ((int64_t*)addr)); - break; - case TYPED_UINT8: - value = PyLong_FromVoidPtr((void*)(Py_ssize_t) * ((uint8_t*)addr)); - break; - case TYPED_UINT16: - value = PyLong_FromVoidPtr((void*)(Py_ssize_t) * ((uint16_t*)addr)); - break; - case TYPED_UINT32: - value = PyLong_FromVoidPtr((void*)(Py_ssize_t) * ((uint32_t*)addr)); - break; - case TYPED_UINT64: - value = PyLong_FromVoidPtr((void*)(Py_ssize_t) * ((uint64_t*)addr)); - break; - case TYPED_DOUBLE: - value = PyFloat_FromDouble(*(double*)addr); - break; - default: - PyErr_SetString(PyExc_RuntimeError, "unsupported field type"); - return NULL; - } - return value; -} - -static inline PyObject* box_primitive(int type, Py_ssize_t value) { - switch (type) { - case TYPED_BOOL: - return PyBool_FromLong((int8_t)value); - case TYPED_INT8: - case TYPED_CHAR: - return PyLong_FromSsize_t((int8_t)value); - case TYPED_INT16: - return PyLong_FromSsize_t((int16_t)value); - case TYPED_INT32: - return PyLong_FromSsize_t((int32_t)value); - case TYPED_INT64: - return PyLong_FromSsize_t((int64_t)value); - case TYPED_UINT8: - return PyLong_FromSize_t((uint8_t)value); - case TYPED_UINT16: - return PyLong_FromSize_t((uint16_t)value); - case TYPED_UINT32: - return PyLong_FromSize_t((uint32_t)value); - case TYPED_UINT64: - return PyLong_FromSize_t((uint64_t)value); - default: - assert(0); - return NULL; - } -} - -static PyObject* invoke_static_function( - PyObject* func, - PyObject** args, - Py_ssize_t nargs, - int awaited) { - return _PyObject_Vectorcall( - func, args, (awaited ? Ci_Py_AWAITED_CALL_MARKER : 0) | nargs, NULL); -} - -int load_method_static_cached_oparg(Py_ssize_t slot, bool is_classmethod) { - return (slot << 1) | (is_classmethod ? 1 : 0); -} - -bool load_method_static_cached_oparg_is_classmethod(int oparg) { - return (oparg & 1) != 0; -} - -Py_ssize_t load_method_static_cached_oparg_slot(int oparg) { - return oparg >> 1; -} - -// Disable UBSAN integer overflow checks etc. as these are not compatible with -// some tests for Static Python which are asserting overflow behavior. -__attribute__((no_sanitize("integer"))) PyObject* _Py_HOT_FUNCTION -Ci_EvalFrame(PyThreadState* tstate, PyFrameObject* f, int throwflag) { - _Py_EnsureTstateNotNULL(tstate); - -#if USE_COMPUTED_GOTOS -/* Import the static jump table */ -#define CINDERX_INTERPRETER -#include "cinderx/Interpreter/cinderx_opcode_targets.h" -#endif - -#ifdef DXPAIRS - int lastopcode = 0; -#endif - PyObject** stack_pointer; /* Next free slot in value stack */ - const _Py_CODEUNIT* next_instr; - int opcode; /* Current opcode */ - int oparg; /* Current opcode argument, if any */ - PyObject **fastlocals, **freevars; - PyObject* retval = NULL; /* Return value */ - _Py_atomic_int* const eval_breaker = &tstate->interp->ceval.eval_breaker; - PyCodeObject* co; - _PyShadowFrame shadow_frame; - - const _Py_CODEUNIT* first_instr; - PyObject* names; - PyObject* consts; - _PyShadow_EvalState shadow = {}; /* facebook T39538061 */ - -#ifdef LLTRACE - _Py_IDENTIFIER(__ltrace__); -#endif - - if (_Py_EnterRecursiveCall(tstate, "")) { - return NULL; - } - - PyTraceInfo trace_info; - /* Mark trace_info as uninitialized */ - trace_info.code = NULL; - - /* WARNING: Because the CFrame lives on the C stack, - * but can be accessed from a heap allocated object (tstate) - * strict stack discipline must be maintained. - */ - CFrame* prev_cframe = tstate->cframe; - trace_info.cframe.use_tracing = prev_cframe->use_tracing; - trace_info.cframe.previous = prev_cframe; - tstate->cframe = &trace_info.cframe; - - /* - * When shadow-frame mode is active, `tstate->frame` may have changed - * between when `f` was allocated and now. Reset `f->f_back` to point to - * the top-most frame if so. - */ - if (f->f_back != tstate->frame) { - Py_XINCREF(tstate->frame); - Py_XSETREF(f->f_back, tstate->frame); - } - - /* push frame */ - tstate->frame = f; - co = f->f_code; - co->co_mutable->curcalls++; - - // Generator shadow frames are managed by the send implementation. - if (f->f_gen == NULL) { - _PyShadowFrame_PushInterp(tstate, &shadow_frame, f); - } - - if (trace_info.cframe.use_tracing) { - if (tstate->c_tracefunc != NULL) { - /* tstate->c_tracefunc, if defined, is a - function that will be called on *every* entry - to a code block. Its return value, if not - None, is a function that will be called at - the start of each executed line of code. - (Actually, the function must return itself - in order to continue tracing.) The trace - functions are called with three arguments: - a pointer to the current frame, a string - indicating why the function is called, and - an argument which depends on the situation. - The global trace function is also called - whenever an exception is detected. */ - if (call_trace_protected( - tstate->c_tracefunc, - tstate->c_traceobj, - tstate, - f, - &trace_info, - PyTrace_CALL, - Py_None)) { - /* Trace function raised an error */ - goto exit_eval_frame; - } - } - if (tstate->c_profilefunc != NULL) { - /* Similar for c_profilefunc, except it needn't - return itself and isn't called for "line" events */ - if (call_trace_protected( - tstate->c_profilefunc, - tstate->c_profileobj, - tstate, - f, - &trace_info, - PyTrace_CALL, - Py_None)) { - /* Profile function raised an error */ - goto exit_eval_frame; - } - } - } - - if (PyDTrace_FUNCTION_ENTRY_ENABLED()) - dtrace_function_entry(f); - - /* facebook begin t39538061 */ - /* Initialize the inline cache after the code object is "hot enough" */ - if (co->co_mutable->shadow == NULL && Ci_cinderx_initialized && - _PyEval_ShadowByteCodeEnabled) { - if (++(co->co_mutable->ncalls) > PYSHADOW_INIT_THRESHOLD) { - if (_PyShadow_InitCache(co) == -1) { - goto error; - } - INLINE_CACHE_CREATED(co->co_mutable); - } - } - /* facebook end t39538061 */ - - names = co->co_names; - consts = co->co_consts; - fastlocals = f->f_localsplus; - freevars = f->f_localsplus + co->co_nlocals; - assert(PyBytes_Check(PyCode_GetCode(co))); - assert(PyBytes_GET_SIZE(PyCode_GetCode(co)) <= INT_MAX); - assert(PyBytes_GET_SIZE(PyCode_GetCode(co)) % sizeof(_Py_CODEUNIT) == 0); - assert(_Py_IS_ALIGNED( - PyBytes_AS_STRING(PyCode_GetCode(co)), sizeof(_Py_CODEUNIT))); - - /* facebook begin t39538061 */ - shadow.code = co; - shadow.first_instr = &first_instr; - assert(PyDict_CheckExact(f->f_builtins)); - PyObject*** global_cache = NULL; - if (co->co_mutable->shadow != NULL && PyDict_CheckExact(f->f_globals)) { - shadow.shadow = co->co_mutable->shadow; - global_cache = shadow.shadow->globals; - first_instr = &shadow.shadow->code[0]; - } else { - first_instr = codeUnit(co); - } - /* facebook end t39538061 */ - - /* - f->f_lasti refers to the index of the last instruction, - unless it's -1 in which case next_instr should be first_instr. - - YIELD_FROM sets f_lasti to itself, in order to repeatedly yield - multiple values. - - When the PREDICT() macros are enabled, some opcode pairs follow in - direct succession without updating f->f_lasti. A successful - prediction effectively links the two codes together as if they - were a single new opcode; accordingly,f->f_lasti will point to - the first code in the pair (for instance, GET_ITER followed by - FOR_ITER is effectively a single opcode and f->f_lasti will point - to the beginning of the combined pair.) - */ - assert(f->f_lasti >= -1); - next_instr = first_instr + f->f_lasti + 1; - stack_pointer = f->f_valuestack + f->f_stackdepth; - /* Set f->f_stackdepth to -1. - * Update when returning or calling trace function. - Having f_stackdepth <= 0 ensures that invalid - values are not visible to the cycle GC. - We choose -1 rather than 0 to assist debugging. - */ - f->f_stackdepth = -1; - f->f_state = FRAME_EXECUTING; - -#ifdef LLTRACE - { - int r = _PyDict_ContainsId(f->f_globals, &PyId___ltrace__); - if (r < 0) { - goto exit_eval_frame; - } - lltrace = r; - } -#endif - - if (throwflag) { /* support for generator.throw() */ - goto error; - } - -#ifdef Py_DEBUG - /* _PyEval_EvalFrameDefault() must not be called with an exception set, - because it can clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!_PyErr_Occurred(tstate)); -#endif - - f->lazy_imports = -1; - f->lazy_imports_cache = 0; - f->lazy_imports_cache_seq = -1; - -main_loop: - for (;;) { - assert(stack_pointer >= f->f_valuestack); /* else underflow */ - assert(STACK_LEVEL() <= co->co_stacksize); /* else overflow */ - assert(!_PyErr_Occurred(tstate)); - - /* Do periodic things. Doing this every time through - the loop would add too much overhead, so we do it - only every Nth instruction. We also do it if - ``pending.calls_to_do'' is set, i.e. when an asynchronous - event needs attention (e.g. a signal handler or - async I/O handler); see Py_AddPendingCall() and - Py_MakePendingCalls() above. */ - - if (_Py_atomic_load_relaxed(eval_breaker)) { - opcode = _Py_OPCODE(*next_instr); - if (opcode != SETUP_FINALLY && opcode != SETUP_WITH && - opcode != BEFORE_ASYNC_WITH && opcode != YIELD_FROM) { - /* Few cases where we skip running signal handlers and other - pending calls: - - If we're about to enter the 'with:'. It will prevent - emitting a resource warning in the common idiom - 'with open(path) as file:'. - - If we're about to enter the 'async with:'. - - If we're about to enter the 'try:' of a try/finally (not - *very* useful, but might help in some cases and it's - traditional) - - If we're resuming a chain of nested 'yield from' or - 'await' calls, then each frame is parked with YIELD_FROM - as its next opcode. If the user hit control-C we want to - wait until we've reached the innermost frame before - running the signal handler and raising KeyboardInterrupt - (see bpo-30039). - */ - if (Cix_eval_frame_handle_pending(tstate) != 0) { - goto error; - } - } - } - - tracing_dispatch: { - int instr_prev = f->f_lasti; - f->f_lasti = INSTR_OFFSET(); - NEXTOPARG(); - - if (PyDTrace_LINE_ENABLED()) - maybe_dtrace_line(f, &trace_info, instr_prev); - - /* line-by-line tracing support */ - - if (trace_info.cframe.use_tracing && tstate->c_tracefunc != NULL && - !tstate->tracing) { - int err; - /* see maybe_call_line_trace() - for expository comments */ - f->f_stackdepth = (int)(stack_pointer - f->f_valuestack); - - err = maybe_call_line_trace( - tstate->c_tracefunc, - tstate->c_traceobj, - tstate, - f, - &trace_info, - instr_prev); - /* Reload possibly changed frame fields */ - JUMPTO(f->f_lasti); - stack_pointer = f->f_valuestack + f->f_stackdepth; - f->f_stackdepth = -1; - if (err) { - /* trace function raised an exception */ - goto error; - } - NEXTOPARG(); - } - } - -#ifdef LLTRACE - /* Instruction tracing */ - - if (lltrace) { - if (HAS_ARG(opcode)) { - printf("%d: %d, %d\n", f->f_lasti, opcode, oparg); - } else { - printf("%d: %d\n", f->f_lasti, opcode); - } - } -#endif -#if USE_COMPUTED_GOTOS == 0 - goto dispatch_opcode; - - predispatch: - if (trace_info.cframe.use_tracing OR_DTRACE_LINE OR_LLTRACE) { - goto tracing_dispatch; - } - f->f_lasti = INSTR_OFFSET(); - NEXTOPARG(); -#endif - dispatch_opcode: -#ifdef DYNAMIC_EXECUTION_PROFILE -#ifdef DXPAIRS - dxpairs[lastopcode][opcode]++; - lastopcode = opcode; -#endif - dxp[opcode]++; -#endif - - switch (opcode) { - /* BEWARE! - It is essential that any operation that fails must goto error - and that all operation that succeed call DISPATCH() ! */ - - case TARGET(NOP): { - DISPATCH(); - } - - case TARGET(LOAD_FAST): { - PyObject* value = GETLOCAL(oparg); - if (value == NULL) { - format_exc_check_arg( - tstate, - PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(PyCode_GetVarnames(co), oparg)); - goto error; - } - Py_INCREF(value); - PUSH(value); - DISPATCH(); - } - - case TARGET(LOAD_CONST): { - PREDICTED(LOAD_CONST); - PyObject* value = GETITEM(consts, oparg); - Py_INCREF(value); - PUSH(value); - DISPATCH(); - } - - case TARGET(STORE_FAST): { - PREDICTED(STORE_FAST); - PyObject* value = POP(); - SETLOCAL(oparg, value); - DISPATCH(); - } - - case TARGET(POP_TOP): { - PyObject* value = POP(); - Py_DECREF(value); - DISPATCH(); - } - - case TARGET(ROT_TWO): { - PyObject* top = TOP(); - PyObject* second = SECOND(); - SET_TOP(second); - SET_SECOND(top); - DISPATCH(); - } - - case TARGET(ROT_THREE): { - PyObject* top = TOP(); - PyObject* second = SECOND(); - PyObject* third = THIRD(); - SET_TOP(second); - SET_SECOND(third); - SET_THIRD(top); - DISPATCH(); - } - - case TARGET(ROT_FOUR): { - PyObject* top = TOP(); - PyObject* second = SECOND(); - PyObject* third = THIRD(); - PyObject* fourth = FOURTH(); - SET_TOP(second); - SET_SECOND(third); - SET_THIRD(fourth); - SET_FOURTH(top); - DISPATCH(); - } - - case TARGET(DUP_TOP): { - PyObject* top = TOP(); - Py_INCREF(top); - PUSH(top); - DISPATCH(); - } - - case TARGET(DUP_TOP_TWO): { - PyObject* top = TOP(); - PyObject* second = SECOND(); - Py_INCREF(top); - Py_INCREF(second); - STACK_GROW(2); - SET_TOP(top); - SET_SECOND(second); - DISPATCH(); - } - - case TARGET(UNARY_POSITIVE): { - PyObject* value = TOP(); - PyObject* res = PyNumber_Positive(value); - Py_DECREF(value); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(UNARY_NEGATIVE): { - PyObject* value = TOP(); - PyObject* res = PyNumber_Negative(value); - Py_DECREF(value); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(UNARY_NOT): { - PyObject* value = TOP(); - int err = PyObject_IsTrue(value); - Py_DECREF(value); - if (err == 0) { - Py_INCREF(Py_True); - SET_TOP(Py_True); - DISPATCH(); - } else if (err > 0) { - Py_INCREF(Py_False); - SET_TOP(Py_False); - DISPATCH(); - } - STACK_SHRINK(1); - goto error; - } - - case TARGET(UNARY_INVERT): { - PyObject* value = TOP(); - PyObject* res = PyNumber_Invert(value); - Py_DECREF(value); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(BINARY_POWER): { - PyObject* exp = POP(); - PyObject* base = TOP(); - PyObject* res = PyNumber_Power(base, exp, Py_None); - Py_DECREF(base); - Py_DECREF(exp); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(BINARY_MULTIPLY): { - PyObject* right = POP(); - PyObject* left = TOP(); - PyObject* res = PyNumber_Multiply(left, right); - Py_DECREF(left); - Py_DECREF(right); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(BINARY_MATRIX_MULTIPLY): { - PyObject* right = POP(); - PyObject* left = TOP(); - PyObject* res = PyNumber_MatrixMultiply(left, right); - Py_DECREF(left); - Py_DECREF(right); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(BINARY_TRUE_DIVIDE): { - PyObject* divisor = POP(); - PyObject* dividend = TOP(); - PyObject* quotient = PyNumber_TrueDivide(dividend, divisor); - Py_DECREF(dividend); - Py_DECREF(divisor); - SET_TOP(quotient); - if (quotient == NULL) - goto error; - DISPATCH(); - } - - case TARGET(BINARY_FLOOR_DIVIDE): { - PyObject* divisor = POP(); - PyObject* dividend = TOP(); - PyObject* quotient = PyNumber_FloorDivide(dividend, divisor); - Py_DECREF(dividend); - Py_DECREF(divisor); - SET_TOP(quotient); - if (quotient == NULL) - goto error; - DISPATCH(); - } - - case TARGET(BINARY_MODULO): { - PyObject* divisor = POP(); - PyObject* dividend = TOP(); - PyObject* res; - if (PyUnicode_CheckExact(dividend) && - (!PyUnicode_Check(divisor) || PyUnicode_CheckExact(divisor))) { - // fast path; string formatting, but not if the RHS is a str subclass - // (see issue28598) - res = PyUnicode_Format(dividend, divisor); - } else { - res = PyNumber_Remainder(dividend, divisor); - } - Py_DECREF(divisor); - Py_DECREF(dividend); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(BINARY_ADD): { - PyObject* right = POP(); - PyObject* left = TOP(); - PyObject* sum; - /* NOTE(vstinner): Please don't try to micro-optimize int+int on - CPython using bytecode, it is simply worthless. - See http://bugs.python.org/issue21955 and - http://bugs.python.org/issue10044 for the discussion. In short, - no patch shown any impact on a realistic benchmark, only a minor - speedup on microbenchmarks. */ - if (PyUnicode_CheckExact(left) && PyUnicode_CheckExact(right)) { - sum = unicode_concatenate(tstate, left, right, f, next_instr); - /* unicode_concatenate consumed the ref to left */ - } else { - sum = PyNumber_Add(left, right); - Py_DECREF(left); - } - Py_DECREF(right); - SET_TOP(sum); - if (sum == NULL) - goto error; - DISPATCH(); - } - - case TARGET(BINARY_SUBTRACT): { - PyObject* right = POP(); - PyObject* left = TOP(); - PyObject* diff = PyNumber_Subtract(left, right); - Py_DECREF(right); - Py_DECREF(left); - SET_TOP(diff); - if (diff == NULL) - goto error; - DISPATCH(); - } - - case TARGET(BINARY_SUBSCR): { - PyObject* res; - PyObject* sub = POP(); - PyObject* container = TOP(); -#ifdef INLINE_CACHE_PROFILE - char type_names[81]; - snprintf( - type_names, - sizeof(type_names), - "%s[%s]", - Py_TYPE(container)->tp_name, - Py_TYPE(sub)->tp_name); - INLINE_CACHE_INCR("binary_subscr_types", type_names); - -#endif - res = shadow.shadow == NULL - ? PyObject_GetItem(container, sub) - : _PyShadow_BinarySubscrWithCache( - &shadow, next_instr, container, sub, oparg); - Py_DECREF(container); - Py_DECREF(sub); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(BINARY_LSHIFT): { - PyObject* right = POP(); - PyObject* left = TOP(); - PyObject* res = PyNumber_Lshift(left, right); - Py_DECREF(left); - Py_DECREF(right); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(BINARY_RSHIFT): { - PyObject* right = POP(); - PyObject* left = TOP(); - PyObject* res = PyNumber_Rshift(left, right); - Py_DECREF(left); - Py_DECREF(right); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(BINARY_AND): { - PyObject* right = POP(); - PyObject* left = TOP(); - PyObject* res = PyNumber_And(left, right); - Py_DECREF(left); - Py_DECREF(right); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(BINARY_XOR): { - PyObject* right = POP(); - PyObject* left = TOP(); - PyObject* res = PyNumber_Xor(left, right); - Py_DECREF(left); - Py_DECREF(right); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(BINARY_OR): { - PyObject* right = POP(); - PyObject* left = TOP(); - PyObject* res = PyNumber_Or(left, right); - Py_DECREF(left); - Py_DECREF(right); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(LIST_APPEND): { - PyObject* v = POP(); - PyObject* list = PEEK(oparg); - int err; - err = Ci_ListOrCheckedList_Append((PyListObject*)list, v); - Py_DECREF(v); - if (err != 0) - goto error; - PREDICT(JUMP_ABSOLUTE); - DISPATCH(); - } - - case TARGET(SET_ADD): { - PyObject* v = POP(); - PyObject* set = PEEK(oparg); - int err; - err = PySet_Add(set, v); - Py_DECREF(v); - if (err != 0) - goto error; - PREDICT(JUMP_ABSOLUTE); - DISPATCH(); - } - - case TARGET(INPLACE_POWER): { - PyObject* exp = POP(); - PyObject* base = TOP(); - PyObject* res = PyNumber_InPlacePower(base, exp, Py_None); - Py_DECREF(base); - Py_DECREF(exp); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(INPLACE_MULTIPLY): { - PyObject* right = POP(); - PyObject* left = TOP(); - PyObject* res = PyNumber_InPlaceMultiply(left, right); - Py_DECREF(left); - Py_DECREF(right); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(INPLACE_MATRIX_MULTIPLY): { - PyObject* right = POP(); - PyObject* left = TOP(); - PyObject* res = PyNumber_InPlaceMatrixMultiply(left, right); - Py_DECREF(left); - Py_DECREF(right); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(INPLACE_TRUE_DIVIDE): { - PyObject* divisor = POP(); - PyObject* dividend = TOP(); - PyObject* quotient = PyNumber_InPlaceTrueDivide(dividend, divisor); - Py_DECREF(dividend); - Py_DECREF(divisor); - SET_TOP(quotient); - if (quotient == NULL) - goto error; - DISPATCH(); - } - - case TARGET(INPLACE_FLOOR_DIVIDE): { - PyObject* divisor = POP(); - PyObject* dividend = TOP(); - PyObject* quotient = PyNumber_InPlaceFloorDivide(dividend, divisor); - Py_DECREF(dividend); - Py_DECREF(divisor); - SET_TOP(quotient); - if (quotient == NULL) - goto error; - DISPATCH(); - } - - case TARGET(INPLACE_MODULO): { - PyObject* right = POP(); - PyObject* left = TOP(); - PyObject* mod = PyNumber_InPlaceRemainder(left, right); - Py_DECREF(left); - Py_DECREF(right); - SET_TOP(mod); - if (mod == NULL) - goto error; - DISPATCH(); - } - - case TARGET(INPLACE_ADD): { - PyObject* right = POP(); - PyObject* left = TOP(); - PyObject* sum; - if (PyUnicode_CheckExact(left) && PyUnicode_CheckExact(right)) { - sum = unicode_concatenate(tstate, left, right, f, next_instr); - /* unicode_concatenate consumed the ref to left */ - } else { - sum = PyNumber_InPlaceAdd(left, right); - Py_DECREF(left); - } - Py_DECREF(right); - SET_TOP(sum); - if (sum == NULL) - goto error; - DISPATCH(); - } - - case TARGET(INPLACE_SUBTRACT): { - PyObject* right = POP(); - PyObject* left = TOP(); - PyObject* diff = PyNumber_InPlaceSubtract(left, right); - Py_DECREF(left); - Py_DECREF(right); - SET_TOP(diff); - if (diff == NULL) - goto error; - DISPATCH(); - } - - case TARGET(INPLACE_LSHIFT): { - PyObject* right = POP(); - PyObject* left = TOP(); - PyObject* res = PyNumber_InPlaceLshift(left, right); - Py_DECREF(left); - Py_DECREF(right); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(INPLACE_RSHIFT): { - PyObject* right = POP(); - PyObject* left = TOP(); - PyObject* res = PyNumber_InPlaceRshift(left, right); - Py_DECREF(left); - Py_DECREF(right); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(INPLACE_AND): { - PyObject* right = POP(); - PyObject* left = TOP(); - PyObject* res = PyNumber_InPlaceAnd(left, right); - Py_DECREF(left); - Py_DECREF(right); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(INPLACE_XOR): { - PyObject* right = POP(); - PyObject* left = TOP(); - PyObject* res = PyNumber_InPlaceXor(left, right); - Py_DECREF(left); - Py_DECREF(right); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(INPLACE_OR): { - PyObject* right = POP(); - PyObject* left = TOP(); - PyObject* res = PyNumber_InPlaceOr(left, right); - Py_DECREF(left); - Py_DECREF(right); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(STORE_SUBSCR): { - PyObject* sub = TOP(); - PyObject* container = SECOND(); - PyObject* v = THIRD(); - int err; - STACK_SHRINK(3); - /* container[sub] = v */ - err = PyObject_SetItem(container, sub, v); - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); - if (err != 0) - goto error; - DISPATCH(); - } - - case TARGET(DELETE_SUBSCR): { - PyObject* sub = TOP(); - PyObject* container = SECOND(); - int err; - STACK_SHRINK(2); - /* del container[sub] */ - err = PyObject_DelItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (err != 0) - goto error; - DISPATCH(); - } - - case TARGET(PRINT_EXPR): { - _Py_IDENTIFIER(displayhook); - PyObject* value = POP(); - PyObject* hook = _PySys_GetObjectId(&PyId_displayhook); - PyObject* res; - if (hook == NULL) { - _PyErr_SetString(tstate, PyExc_RuntimeError, "lost sys.displayhook"); - Py_DECREF(value); - goto error; - } - res = PyObject_CallOneArg(hook, value); - Py_DECREF(value); - if (res == NULL) - goto error; - Py_DECREF(res); - DISPATCH(); - } - - case TARGET(RAISE_VARARGS): { - PyObject *cause = NULL, *exc = NULL; - switch (oparg) { - case 2: - cause = POP(); /* cause */ - __attribute__((fallthrough)); - case 1: - exc = POP(); /* exc */ - __attribute__((fallthrough)); - case 0: - if (do_raise(tstate, exc, cause)) { - goto exception_unwind; - } - break; - default: - _PyErr_SetString( - tstate, PyExc_SystemError, "bad RAISE_VARARGS oparg"); - break; - } - goto error; - } - - case TARGET(RETURN_VALUE): { - retval = POP(); - assert(f->f_iblock == 0); - assert(EMPTY()); - f->f_state = FRAME_RETURNED; - f->f_stackdepth = 0; - goto exiting; - } - - case TARGET(GET_AITER): { - PyObject* obj = TOP(); - PyObject* iter = Ci_GetAIter(tstate, obj); - Py_DECREF(obj); - SET_TOP(iter); - if (iter == NULL) { - goto error; - } - DISPATCH(); - } - - case TARGET(GET_ANEXT): { - PyObject* awaitable = Ci_GetANext(tstate, TOP()); - if (awaitable == NULL) { - goto error; - } - PUSH(awaitable); - PREDICT(LOAD_CONST); - DISPATCH(); - } - - case TARGET(GET_AWAITABLE): { - PREDICTED(GET_AWAITABLE); - PyObject* iterable = TOP(); - PyObject* iter = Cix_PyCoro_GetAwaitableIter(iterable); - - if (iter == NULL) { - int opcode_at_minus_3 = 0; - if ((next_instr - first_instr) > 2) { - opcode_at_minus_3 = _Py_OPCODE(next_instr[-3]); - } - format_awaitable_error( - tstate, - Py_TYPE(iterable), - opcode_at_minus_3, - _Py_OPCODE(next_instr[-2])); - } - - Py_DECREF(iterable); - - if (iter != NULL && PyCoro_CheckExact(iter)) { - PyObject* yf = Cix_PyGen_yf((PyGenObject*)iter); - if (yf != NULL) { - /* `iter` is a coroutine object that is being - awaited, `yf` is a pointer to the current awaitable - being awaited on. */ - Py_DECREF(yf); - Py_CLEAR(iter); - _PyErr_SetString( - tstate, - PyExc_RuntimeError, - "coroutine is being awaited already"); - /* The code below jumps to `error` if `iter` is NULL. */ - } - } - - SET_TOP(iter); /* Even if it's NULL */ - - if (iter == NULL) { - goto error; - } - - PREDICT(LOAD_CONST); - DISPATCH(); - } - - case TARGET(YIELD_FROM): { - PyObject* v = POP(); - PyObject* receiver = TOP(); - PySendResult gen_status; - if (f->f_gen && (co->co_flags & CO_COROUTINE)) { - Ci_PyAwaitable_SetAwaiter(receiver, f->f_gen); - } - if (tstate->c_tracefunc == NULL) { - gen_status = PyIter_Send(receiver, v, &retval); - } else { - _Py_IDENTIFIER(send); - if (Py_IsNone(v) && PyIter_Check(receiver)) { - retval = Py_TYPE(receiver)->tp_iternext(receiver); - } else { - retval = _PyObject_CallMethodIdOneArg(receiver, &PyId_send, v); - } - if (retval == NULL) { - if (tstate->c_tracefunc != NULL && - _PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) - call_exc_trace( - tstate->c_tracefunc, - tstate->c_traceobj, - tstate, - f, - &trace_info); - if (_PyGen_FetchStopIterationValue(&retval) == 0) { - gen_status = PYGEN_RETURN; - } else { - gen_status = PYGEN_ERROR; - } - } else { - gen_status = PYGEN_NEXT; - } - } - Py_DECREF(v); - if (gen_status == PYGEN_ERROR) { - assert(retval == NULL); - goto error; - } - if (gen_status == PYGEN_RETURN) { - assert(retval != NULL); - - Py_DECREF(receiver); - SET_TOP(retval); - retval = NULL; - DISPATCH(); - } - assert(gen_status == PYGEN_NEXT); - /* receiver remains on stack, retval is value to be yielded */ - /* and repeat... */ - assert(f->f_lasti > 0); - f->f_lasti -= 1; - f->f_state = FRAME_SUSPENDED; - f->f_stackdepth = (int)(stack_pointer - f->f_valuestack); - goto exiting; - } - - case TARGET(YIELD_VALUE): { - retval = POP(); - - if (co->co_flags & CO_ASYNC_GENERATOR) { - PyObject* w = Cix_PyAsyncGenValueWrapperNew(retval); - Py_DECREF(retval); - if (w == NULL) { - retval = NULL; - goto error; - } - retval = w; - } - f->f_state = FRAME_SUSPENDED; - f->f_stackdepth = (int)(stack_pointer - f->f_valuestack); - goto exiting; - } - - case TARGET(GEN_START): { - PyObject* none = POP(); - assert(none == Py_None); - assert(oparg < 3); - Py_DECREF(none); - DISPATCH(); - } - - case TARGET(POP_EXCEPT): { - PyObject *type, *value, *traceback; - _PyErr_StackItem* exc_info; - PyTryBlock* b = PyFrame_BlockPop(f); - if (b->b_type != EXCEPT_HANDLER) { - _PyErr_SetString( - tstate, - PyExc_SystemError, - "popped block is not an except handler"); - goto error; - } - assert( - STACK_LEVEL() >= (b)->b_level + 3 && - STACK_LEVEL() <= (b)->b_level + 4); - exc_info = tstate->exc_info; - type = exc_info->exc_type; - value = exc_info->exc_value; - traceback = exc_info->exc_traceback; - exc_info->exc_type = POP(); - exc_info->exc_value = POP(); - exc_info->exc_traceback = POP(); - Py_XDECREF(type); - Py_XDECREF(value); - Py_XDECREF(traceback); - DISPATCH(); - } - - case TARGET(POP_BLOCK): { - PyFrame_BlockPop(f); - DISPATCH(); - } - - case TARGET(RERAISE): { - assert(f->f_iblock > 0); - if (oparg) { - f->f_lasti = f->f_blockstack[f->f_iblock - 1].b_handler; - } - PyObject* exc = POP(); - PyObject* val = POP(); - PyObject* tb = POP(); - assert(PyExceptionClass_Check(exc)); - _PyErr_Restore(tstate, exc, val, tb); - goto exception_unwind; - } - - case TARGET(END_ASYNC_FOR): { - PyObject* exc = POP(); - assert(PyExceptionClass_Check(exc)); - if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { - PyTryBlock* b = PyFrame_BlockPop(f); - assert(b->b_type == EXCEPT_HANDLER); - Py_DECREF(exc); - UNWIND_EXCEPT_HANDLER(b); - Py_DECREF(POP()); - JUMPBY(oparg); - DISPATCH(); - } else { - PyObject* val = POP(); - PyObject* tb = POP(); - _PyErr_Restore(tstate, exc, val, tb); - goto exception_unwind; - } - } - - case TARGET(LOAD_ASSERTION_ERROR): { - PyObject* value = PyExc_AssertionError; - Py_INCREF(value); - PUSH(value); - DISPATCH(); - } - - case TARGET(LOAD_BUILD_CLASS): { - _Py_IDENTIFIER(__build_class__); - - PyObject* bc; - if (PyDict_CheckExact(f->f_builtins)) { - bc = _PyDict_GetItemIdWithError(f->f_builtins, &PyId___build_class__); - if (bc == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString( - tstate, PyExc_NameError, "__build_class__ not found"); - } - goto error; - } - Py_INCREF(bc); - } else { - PyObject* build_class_str = _PyUnicode_FromId(&PyId___build_class__); - if (build_class_str == NULL) - goto error; - bc = PyObject_GetItem(f->f_builtins, build_class_str); - if (bc == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) - _PyErr_SetString( - tstate, PyExc_NameError, "__build_class__ not found"); - goto error; - } - } - PUSH(bc); - DISPATCH(); - } - - case TARGET(STORE_NAME): { - PyObject* name = GETITEM(names, oparg); - PyObject* v = POP(); - PyObject* ns = f->f_locals; - int err; - if (ns == NULL) { - _PyErr_Format( - tstate, - PyExc_SystemError, - "no locals found when storing %R", - name); - Py_DECREF(v); - goto error; - } - if (PyDict_CheckExact(ns)) { - err = PyDict_SetItem(ns, name, v); - } else { - err = PyObject_SetItem(ns, name, v); - } - Py_DECREF(v); - if (err != 0) - goto error; - DISPATCH(); - } - - case TARGET(DELETE_NAME): { - PyObject* name = GETITEM(names, oparg); - PyObject* ns = f->f_locals; - int err; - if (ns == NULL) { - _PyErr_Format( - tstate, PyExc_SystemError, "no locals when deleting %R", name); - goto error; - } - err = PyObject_DelItem(ns, name); - if (err != 0) { - format_exc_check_arg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - goto error; - } - DISPATCH(); - } - - case TARGET(UNPACK_SEQUENCE): { - PREDICTED(UNPACK_SEQUENCE); - PyObject *seq = POP(), *item, **items; - if (PyTuple_CheckExact(seq) && PyTuple_GET_SIZE(seq) == oparg) { - items = ((PyTupleObject*)seq)->ob_item; - while (oparg--) { - item = items[oparg]; - Py_INCREF(item); - PUSH(item); - } - } else if (PyList_CheckExact(seq) && PyList_GET_SIZE(seq) == oparg) { - items = ((PyListObject*)seq)->ob_item; - while (oparg--) { - item = items[oparg]; - Py_INCREF(item); - PUSH(item); - } - } else if (unpack_iterable( - tstate, seq, oparg, -1, stack_pointer + oparg)) { - STACK_GROW(oparg); - } else { - /* unpack_iterable() raised an exception */ - Py_DECREF(seq); - goto error; - } - Py_DECREF(seq); - DISPATCH(); - } - - case TARGET(UNPACK_EX): { - int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); - PyObject* seq = POP(); - - if (unpack_iterable( - tstate, - seq, - oparg & 0xFF, - oparg >> 8, - stack_pointer + totalargs)) { - stack_pointer += totalargs; - } else { - Py_DECREF(seq); - goto error; - } - Py_DECREF(seq); - DISPATCH(); - } - - case TARGET(STORE_ATTR): { - PyObject* name = GETITEM(names, oparg); - PyObject* owner = TOP(); - PyObject* v = SECOND(); -#ifdef INLINE_CACHE_PROFILE - _PyShadow_LogLocation(&shadow, next_instr, "STORE_ATTR"); - char type_name[81]; - snprintf( - type_name, - sizeof(type_name), - "STORE_ATTR_TYPE[%s]", - Py_TYPE(owner)->tp_name); - _PyShadow_LogLocation(&shadow, next_instr, type_name); -#endif - int err; - STACK_SHRINK(2); - err = shadow.shadow == NULL - ? PyObject_SetAttr(owner, name, v) - : _PyShadow_StoreAttrWithCache(&shadow, next_instr, owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); - if (err != 0) - goto error; - DISPATCH(); - } - - case TARGET(DELETE_ATTR): { - PyObject* name = GETITEM(names, oparg); - PyObject* owner = POP(); - int err; - err = PyObject_SetAttr(owner, name, (PyObject*)NULL); - Py_DECREF(owner); - if (err != 0) - goto error; - DISPATCH(); - } - - case TARGET(STORE_GLOBAL): { - PyObject* name = GETITEM(names, oparg); - PyObject* v = POP(); - int err; - err = PyDict_SetItem(f->f_globals, name, v); - Py_DECREF(v); - if (err != 0) - goto error; - DISPATCH(); - } - - case TARGET(DELETE_GLOBAL): { - PyObject* name = GETITEM(names, oparg); - int err; - err = PyDict_DelItem(f->f_globals, name); - if (err != 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - format_exc_check_arg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - } - goto error; - } - DISPATCH(); - } - - case TARGET(LOAD_NAME): { - PyObject* name = GETITEM(names, oparg); - PyObject* locals = f->f_locals; - PyObject* v; - if (locals == NULL) { - _PyErr_Format( - tstate, PyExc_SystemError, "no locals when loading %R", name); - goto error; - } - if (PyDict_CheckExact(locals)) { - v = PyDict_GetItemWithError(locals, name); - if (v != NULL) { - Py_INCREF(v); - } else if (_PyErr_Occurred(tstate)) { - goto error; - } - } else { - v = PyObject_GetItem(locals, name); - if (v == NULL) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) - goto error; - _PyErr_Clear(tstate); - } - } - if (v == NULL) { - v = PyDict_GetItemWithError(f->f_globals, name); - if (v != NULL) { - Py_INCREF(v); - } else if (_PyErr_Occurred(tstate)) { - goto error; - } else { - if (PyDict_CheckExact(f->f_builtins)) { - v = PyDict_GetItemWithError(f->f_builtins, name); - if (v == NULL) { - if (!_PyErr_Occurred(tstate)) { - format_exc_check_arg( - tstate, PyExc_NameError, NAME_ERROR_MSG, name); - } - goto error; - } - Py_INCREF(v); - } else { - v = PyObject_GetItem(f->f_builtins, name); - if (v == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - format_exc_check_arg( - tstate, PyExc_NameError, NAME_ERROR_MSG, name); - } - goto error; - } - } - } - } - PUSH(v); - DISPATCH(); - } - - case TARGET(LOAD_GLOBAL): { - PyObject* name; - PyObject* v; - if (PyDict_CheckExact(f->f_globals)) { - assert(PyDict_CheckExact(f->f_builtins)); - name = GETITEM(names, oparg); - v = Cix_PyDict_LoadGlobal( - (PyDictObject*)f->f_globals, (PyDictObject*)f->f_builtins, name); - if (v == NULL) { - if (!_PyErr_Occurred(tstate)) { - /* Cix_PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ - format_exc_check_arg( - tstate, PyExc_NameError, NAME_ERROR_MSG, name); - } - goto error; - } - - if (shadow.shadow != NULL) { - _PyShadow_InitGlobal( - &shadow, next_instr, f->f_globals, f->f_builtins, name); - } - - Py_INCREF(v); - /* facebook end */ - } else { - /* Slow-path if globals or builtins is not a dict */ - - /* namespace 1: globals */ - name = GETITEM(names, oparg); - v = PyObject_GetItem(f->f_globals, name); - if (v == NULL) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - goto error; - } - _PyErr_Clear(tstate); - - /* namespace 2: builtins */ - v = PyObject_GetItem(f->f_builtins, name); - if (v == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - format_exc_check_arg( - tstate, PyExc_NameError, NAME_ERROR_MSG, name); - } - goto error; - } - } - } - PUSH(v); - DISPATCH(); - } - - case TARGET(DELETE_FAST): { - PyObject* v = GETLOCAL(oparg); - if (v != NULL) { - SETLOCAL(oparg, NULL); - } - DISPATCH(); - } - - case TARGET(DELETE_DEREF): { - PyObject* cell = freevars[oparg]; - PyObject* oldobj = PyCell_GET(cell); - if (oldobj != NULL) { - PyCell_SET(cell, NULL); - Py_DECREF(oldobj); - DISPATCH(); - } - format_exc_unbound(tstate, co, oparg); - goto error; - } - - case TARGET(LOAD_CLOSURE): { - PyObject* cell = freevars[oparg]; - Py_INCREF(cell); - PUSH(cell); - DISPATCH(); - } - - case TARGET(LOAD_CLASSDEREF): { - PyObject *name, *value, *locals = f->f_locals; - Py_ssize_t idx; - assert(locals); - assert(oparg >= PyTuple_GET_SIZE(PyCode_GetCellvars(co))); - idx = oparg - PyTuple_GET_SIZE(PyCode_GetCellvars(co)); - assert(idx >= 0 && idx < PyTuple_GET_SIZE(PyCode_GetFreevars(co))); - name = PyTuple_GET_ITEM(PyCode_GetFreevars(co), idx); - if (PyDict_CheckExact(locals)) { - value = PyDict_GetItemWithError(locals, name); - if (value != NULL) { - Py_INCREF(value); - } else if (_PyErr_Occurred(tstate)) { - goto error; - } - } else { - value = PyObject_GetItem(locals, name); - if (value == NULL) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - goto error; - } - _PyErr_Clear(tstate); - } - } - if (!value) { - PyObject* cell = freevars[oparg]; - value = PyCell_GET(cell); - if (value == NULL) { - format_exc_unbound(tstate, co, oparg); - goto error; - } - Py_INCREF(value); - } - PUSH(value); - DISPATCH(); - } - - case TARGET(LOAD_DEREF): { - PyObject* cell = freevars[oparg]; - PyObject* value = PyCell_GET(cell); - if (value == NULL) { - format_exc_unbound(tstate, co, oparg); - goto error; - } - Py_INCREF(value); - PUSH(value); - DISPATCH(); - } - - case TARGET(STORE_DEREF): { - PyObject* v = POP(); - PyObject* cell = freevars[oparg]; - PyObject* oldobj = PyCell_GET(cell); - PyCell_SET(cell, v); - Py_XDECREF(oldobj); - DISPATCH(); - } - - case TARGET(BUILD_STRING): { - PyObject* str; - PyObject* empty = PyUnicode_New(0, 0); - if (empty == NULL) { - goto error; - } - str = _PyUnicode_JoinArray(empty, stack_pointer - oparg, oparg); - Py_DECREF(empty); - if (str == NULL) - goto error; - while (--oparg >= 0) { - PyObject* item = POP(); - Py_DECREF(item); - } - PUSH(str); - DISPATCH(); - } - - case TARGET(BUILD_TUPLE): { - PyObject* tup = PyTuple_New(oparg); - if (tup == NULL) - goto error; - while (--oparg >= 0) { - PyObject* item = POP(); - PyTuple_SET_ITEM(tup, oparg, item); - } - PUSH(tup); - DISPATCH(); - } - - case TARGET(BUILD_LIST): { - PyObject* list = PyList_New(oparg); - if (list == NULL) - goto error; - while (--oparg >= 0) { - PyObject* item = POP(); - PyList_SET_ITEM(list, oparg, item); - } - PUSH(list); - DISPATCH(); - } - - case TARGET(LIST_TO_TUPLE): { - PyObject* list = POP(); - PyObject* tuple = PyList_AsTuple(list); - Py_DECREF(list); - if (tuple == NULL) { - goto error; - } - PUSH(tuple); - DISPATCH(); - } - - case TARGET(LIST_EXTEND): { - PyObject* iterable = POP(); - PyObject* list = PEEK(oparg); - PyObject* none_val = _PyList_Extend((PyListObject*)list, iterable); - if (none_val == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && - (Py_TYPE(iterable)->tp_iter == NULL && - !PySequence_Check(iterable))) { - _PyErr_Clear(tstate); - _PyErr_Format( - tstate, - PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); - } - Py_DECREF(iterable); - goto error; - } - Py_DECREF(none_val); - Py_DECREF(iterable); - DISPATCH(); - } - - case TARGET(SET_UPDATE): { - PyObject* iterable = POP(); - PyObject* set = PEEK(oparg); - int err = _PySet_Update(set, iterable); - Py_DECREF(iterable); - if (err < 0) { - goto error; - } - DISPATCH(); - } - - case TARGET(BUILD_SET): { - PyObject* set = PySet_New(NULL); - int err = 0; - int i; - if (set == NULL) - goto error; - for (i = oparg; i > 0; i--) { - PyObject* item = PEEK(i); - if (err == 0) - err = PySet_Add(set, item); - Py_DECREF(item); - } - STACK_SHRINK(oparg); - if (err != 0) { - Py_DECREF(set); - goto error; - } - PUSH(set); - DISPATCH(); - } - -#define Ci_BUILD_DICT(map_size, set_item) \ - \ - for (Py_ssize_t i = map_size; i > 0; i--) { \ - int err; \ - PyObject* key = PEEK(2 * i); \ - PyObject* value = PEEK(2 * i - 1); \ - err = set_item(map, key, value); \ - if (err != 0) { \ - Py_DECREF(map); \ - goto error; \ - } \ - } \ - \ - while (map_size--) { \ - Py_DECREF(POP()); \ - Py_DECREF(POP()); \ - } \ - PUSH(map); - - case TARGET(BUILD_MAP): { - PyObject* map = _PyDict_NewPresized((Py_ssize_t)oparg); - if (map == NULL) - goto error; - - Ci_BUILD_DICT(oparg, Ci_DictOrChecked_SetItem); - - DISPATCH(); - } - - case TARGET(SETUP_ANNOTATIONS): { - _Py_IDENTIFIER(__annotations__); - int err; - PyObject* ann_dict; - if (f->f_locals == NULL) { - _PyErr_Format( - tstate, - PyExc_SystemError, - "no locals found when setting up annotations"); - goto error; - } - /* check if __annotations__ in locals()... */ - if (PyDict_CheckExact(f->f_locals)) { - ann_dict = - _PyDict_GetItemIdWithError(f->f_locals, &PyId___annotations__); - if (ann_dict == NULL) { - if (_PyErr_Occurred(tstate)) { - goto error; - } - /* ...if not, create a new one */ - ann_dict = PyDict_New(); - if (ann_dict == NULL) { - goto error; - } - err = - _PyDict_SetItemId(f->f_locals, &PyId___annotations__, ann_dict); - Py_DECREF(ann_dict); - if (err != 0) { - goto error; - } - } - } else { - /* do the same if locals() is not a dict */ - PyObject* ann_str = _PyUnicode_FromId(&PyId___annotations__); - if (ann_str == NULL) { - goto error; - } - ann_dict = PyObject_GetItem(f->f_locals, ann_str); - if (ann_dict == NULL) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - goto error; - } - _PyErr_Clear(tstate); - ann_dict = PyDict_New(); - if (ann_dict == NULL) { - goto error; - } - err = PyObject_SetItem(f->f_locals, ann_str, ann_dict); - Py_DECREF(ann_dict); - if (err != 0) { - goto error; - } - } else { - Py_DECREF(ann_dict); - } - } - DISPATCH(); - } - - case TARGET(BUILD_CONST_KEY_MAP): { - Py_ssize_t i; - PyObject* map; - PyObject* keys = TOP(); - if (!PyTuple_CheckExact(keys) || - PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { - _PyErr_SetString( - tstate, - PyExc_SystemError, - "bad BUILD_CONST_KEY_MAP keys argument"); - goto error; - } - map = _PyDict_NewPresized((Py_ssize_t)oparg); - if (map == NULL) { - goto error; - } - for (i = oparg; i > 0; i--) { - int err; - PyObject* key = PyTuple_GET_ITEM(keys, oparg - i); - PyObject* value = PEEK(i + 1); - err = PyDict_SetItem(map, key, value); - if (err != 0) { - Py_DECREF(map); - goto error; - } - } - - Py_DECREF(POP()); - while (oparg--) { - Py_DECREF(POP()); - } - PUSH(map); - DISPATCH(); - } - - case TARGET(DICT_UPDATE): { - PyObject* update = POP(); - PyObject* dict = PEEK(oparg); - if (PyDict_Update(dict, update) < 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { - _PyErr_Format( - tstate, - PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); - } - Py_DECREF(update); - goto error; - } - Py_DECREF(update); - DISPATCH(); - } - - case TARGET(DICT_MERGE): { - PyObject* update = POP(); - PyObject* dict = PEEK(oparg); - - if (_PyDict_MergeEx(dict, update, 2) < 0) { - format_kwargs_error(tstate, PEEK(2 + oparg), update); - Py_DECREF(update); - goto error; - } - Py_DECREF(update); - PREDICT(CALL_FUNCTION_EX); - DISPATCH(); - } - - case TARGET(MAP_ADD): { - PyObject* value = TOP(); - PyObject* key = SECOND(); - PyObject* map; - int err; - STACK_SHRINK(2); - map = PEEK(oparg); /* dict */ - assert(PyDict_CheckExact(map) || Ci_CheckedDict_Check(map)); - err = Ci_DictOrChecked_SetItem(map, key, value); /* map[key] = value */ - Py_DECREF(value); - Py_DECREF(key); - if (err != 0) - goto error; - PREDICT(JUMP_ABSOLUTE); - DISPATCH(); - } - - case TARGET(LOAD_ATTR): { - PyObject* name = GETITEM(names, oparg); - PyObject* owner = TOP(); - PyObject* res = shadow.shadow == NULL - ? PyObject_GetAttr(owner, name) - : _PyShadow_LoadAttrWithCache(&shadow, next_instr, owner, name); - Py_DECREF(owner); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(COMPARE_OP): { - assert(oparg <= Py_GE); - PyObject* right = POP(); - PyObject* left = TOP(); - PyObject* res = PyObject_RichCompare(left, right, oparg); - SET_TOP(res); - Py_DECREF(left); - Py_DECREF(right); - if (res == NULL) - goto error; - PREDICT(POP_JUMP_IF_FALSE); - PREDICT(POP_JUMP_IF_TRUE); - DISPATCH(); - } - - case TARGET(IS_OP): { - PyObject* right = POP(); - PyObject* left = TOP(); - int res = Py_Is(left, right) ^ oparg; - PyObject* b = res ? Py_True : Py_False; - Py_INCREF(b); - SET_TOP(b); - Py_DECREF(left); - Py_DECREF(right); - PREDICT(POP_JUMP_IF_FALSE); - PREDICT(POP_JUMP_IF_TRUE); - DISPATCH(); - } - - case TARGET(CONTAINS_OP): { - PyObject* right = POP(); - PyObject* left = POP(); - int res = PySequence_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); - if (res < 0) { - goto error; - } - PyObject* b = (res ^ oparg) ? Py_True : Py_False; - Py_INCREF(b); - PUSH(b); - PREDICT(POP_JUMP_IF_FALSE); - PREDICT(POP_JUMP_IF_TRUE); - DISPATCH(); - } - -#define CANNOT_CATCH_MSG \ - "catching classes that do not inherit from " \ - "BaseException is not allowed" - - case TARGET(JUMP_IF_NOT_EXC_MATCH): { - PyObject* right = POP(); - PyObject* left = POP(); - if (PyTuple_Check(right)) { - Py_ssize_t i, length; - length = PyTuple_GET_SIZE(right); - for (i = 0; i < length; i++) { - PyObject* exc = PyTuple_GET_ITEM(right, i); - if (!PyExceptionClass_Check(exc)) { - _PyErr_SetString(tstate, PyExc_TypeError, CANNOT_CATCH_MSG); - Py_DECREF(left); - Py_DECREF(right); - goto error; - } - } - } else { - if (!PyExceptionClass_Check(right)) { - _PyErr_SetString(tstate, PyExc_TypeError, CANNOT_CATCH_MSG); - Py_DECREF(left); - Py_DECREF(right); - goto error; - } - } - int res = PyErr_GivenExceptionMatches(left, right); - Py_DECREF(left); - Py_DECREF(right); - if (res > 0) { - /* Exception matches -- Do nothing */; - } else if (res == 0) { - JUMPTO(oparg); - } else { - goto error; - } - DISPATCH(); - } - - case TARGET(IMPORT_NAME): { - PyObject* name = GETITEM(names, oparg); - PyObject* fromlist = POP(); - PyObject* level = TOP(); - PyObject* res; - - if (f->f_globals == f->f_locals && f->f_iblock == 0 && - _PyImport_IsLazyImportsActive(tstate)) { - res = _PyImport_LazyImportName( - f->f_builtins, - f->f_globals, - f->f_locals == NULL ? Py_None : f->f_locals, - name, - fromlist, - level); - } else { - res = _PyImport_ImportName( - f->f_builtins, - f->f_globals, - f->f_locals == NULL ? Py_None : f->f_locals, - name, - fromlist, - level); - } - - Py_DECREF(level); - Py_DECREF(fromlist); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(IMPORT_STAR): { - PyObject *from = POP(), *locals; - int err; - if (PyLazyImport_CheckExact(from)) { - PyObject* mod = _PyImport_LoadLazyImportTstate(tstate, from, 1); - Py_DECREF(from); - if (mod == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString(tstate, PyExc_SystemError, "Lazy Import cycle"); - } - goto error; - } - from = mod; - } - - if (PyFrame_FastToLocalsWithError(f) < 0) { - Py_DECREF(from); - goto error; - } - - locals = f->f_locals; - if (locals == NULL) { - _PyErr_SetString( - tstate, PyExc_SystemError, "no locals found during 'import *'"); - Py_DECREF(from); - goto error; - } - err = import_all_from(tstate, locals, from); - Py_DECREF(from); - if (err != 0) - goto error; - PyFrame_LocalsToFast(f, 0); - DISPATCH(); - } - - case TARGET(IMPORT_FROM): { - PyObject* name = GETITEM(names, oparg); - PyObject* from = TOP(); - PyObject* res; - if (PyLazyImport_CheckExact(from)) { - res = _PyImport_LazyImportFrom(tstate, from, name); - } else { - res = _PyImport_ImportFrom(tstate, from, name); - } - PUSH(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(JUMP_FORWARD): { - JUMPBY(oparg); - DISPATCH(); - } - - case TARGET(POP_JUMP_IF_FALSE): { - PREDICTED(POP_JUMP_IF_FALSE); - PyObject* cond = POP(); - int err; - if (Py_IsTrue(cond)) { - Py_DECREF(cond); - DISPATCH(); - } - if (Py_IsFalse(cond)) { - Py_DECREF(cond); - JUMPTO(oparg); - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - err = PyObject_IsTrue(cond); - Py_DECREF(cond); - if (err > 0) - ; - else if (err == 0) { - JUMPTO(oparg); - CHECK_EVAL_BREAKER(); - } else - goto error; - DISPATCH(); - } - - case TARGET(POP_JUMP_IF_TRUE): { - PREDICTED(POP_JUMP_IF_TRUE); - PyObject* cond = POP(); - int err; - if (Py_IsFalse(cond)) { - Py_DECREF(cond); - DISPATCH(); - } - if (Py_IsTrue(cond)) { - Py_DECREF(cond); - JUMPTO(oparg); - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - err = PyObject_IsTrue(cond); - Py_DECREF(cond); - if (err > 0) { - JUMPTO(oparg); - CHECK_EVAL_BREAKER(); - } else if (err == 0) - ; - else - goto error; - DISPATCH(); - } - - case TARGET(JUMP_IF_FALSE_OR_POP): { - PyObject* cond = TOP(); - int err; - if (Py_IsTrue(cond)) { - STACK_SHRINK(1); - Py_DECREF(cond); - DISPATCH(); - } - if (Py_IsFalse(cond)) { - JUMPTO(oparg); - DISPATCH(); - } - err = PyObject_IsTrue(cond); - if (err > 0) { - STACK_SHRINK(1); - Py_DECREF(cond); - } else if (err == 0) - JUMPTO(oparg); - else - goto error; - DISPATCH(); - } - - case TARGET(JUMP_IF_TRUE_OR_POP): { - PyObject* cond = TOP(); - int err; - if (Py_IsFalse(cond)) { - STACK_SHRINK(1); - Py_DECREF(cond); - DISPATCH(); - } - if (Py_IsTrue(cond)) { - JUMPTO(oparg); - DISPATCH(); - } - err = PyObject_IsTrue(cond); - if (err > 0) { - JUMPTO(oparg); - } else if (err == 0) { - STACK_SHRINK(1); - Py_DECREF(cond); - } else - goto error; - DISPATCH(); - } - - case TARGET(JUMP_ABSOLUTE): { - PREDICTED(JUMP_ABSOLUTE); - JUMPTO(oparg); - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - case TARGET(GET_LEN): { - // PUSH(len(TOS)) - Py_ssize_t len_i = PyObject_Length(TOP()); - if (len_i < 0) { - goto error; - } - PyObject* len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) { - goto error; - } - PUSH(len_o); - DISPATCH(); - } - - case TARGET(MATCH_CLASS): { - // Pop TOS. On success, set TOS to True and TOS1 to a tuple of - // attributes. On failure, set TOS to False. - PyObject* names = POP(); - PyObject* type = TOP(); - PyObject* subject = SECOND(); - assert(PyTuple_CheckExact(names)); - PyObject* attrs = match_class(tstate, subject, type, oparg, names); - Py_DECREF(names); - if (attrs) { - // Success! - assert(PyTuple_CheckExact(attrs)); - Py_DECREF(subject); - SET_SECOND(attrs); - } else if (_PyErr_Occurred(tstate)) { - goto error; - } - Py_DECREF(type); - SET_TOP(PyBool_FromLong(!!attrs)); - DISPATCH(); - } - - case TARGET(MATCH_MAPPING): { - PyObject* subject = TOP(); - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - PyObject* res = match ? Py_True : Py_False; - Py_INCREF(res); - PUSH(res); - DISPATCH(); - } - - case TARGET(MATCH_SEQUENCE): { - PyObject* subject = TOP(); - int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - PyObject* res = match ? Py_True : Py_False; - Py_INCREF(res); - PUSH(res); - DISPATCH(); - } - - case TARGET(MATCH_KEYS): { - // On successful match for all keys, PUSH(values) and PUSH(True). - // Otherwise, PUSH(None) and PUSH(False). - PyObject* keys = TOP(); - PyObject* subject = SECOND(); - PyObject* values_or_none = match_keys(tstate, subject, keys); - if (values_or_none == NULL) { - goto error; - } - PUSH(values_or_none); - if (Py_IsNone(values_or_none)) { - Py_INCREF(Py_False); - PUSH(Py_False); - DISPATCH(); - } - assert(PyTuple_CheckExact(values_or_none)); - Py_INCREF(Py_True); - PUSH(Py_True); - DISPATCH(); - } - - case TARGET(COPY_DICT_WITHOUT_KEYS): { - // rest = dict(TOS1) - // for key in TOS: - // del rest[key] - // SET_TOP(rest) - PyObject* keys = TOP(); - PyObject* subject = SECOND(); - PyObject* rest = PyDict_New(); - if (rest == NULL || PyDict_Update(rest, subject)) { - Py_XDECREF(rest); - goto error; - } - // This may seem a bit inefficient, but keys is rarely big enough to - // actually impact runtime. - assert(PyTuple_CheckExact(keys)); - for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(keys); i++) { - if (PyDict_DelItem(rest, PyTuple_GET_ITEM(keys, i))) { - Py_DECREF(rest); - goto error; - } - } - Py_DECREF(keys); - SET_TOP(rest); - DISPATCH(); - } - - case TARGET(GET_ITER): { - /* before: [obj]; after [getiter(obj)] */ - PyObject* iterable = TOP(); - PyObject* iter = PyObject_GetIter(iterable); - Py_DECREF(iterable); - SET_TOP(iter); - if (iter == NULL) - goto error; - PREDICT(FOR_ITER); - PREDICT(CALL_FUNCTION); - DISPATCH(); - } - - case TARGET(GET_YIELD_FROM_ITER): { - /* before: [obj]; after [getiter(obj)] */ - PyObject* iterable = TOP(); - PyObject* iter; - if (PyCoro_CheckExact(iterable)) { - /* `iterable` is a coroutine */ - if (!(co->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - /* and it is used in a 'yield from' expression of a - regular generator. */ - Py_DECREF(iterable); - SET_TOP(NULL); - _PyErr_SetString( - tstate, - PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "in a non-coroutine generator"); - goto error; - } - } else if (!PyGen_CheckExact(iterable)) { - /* `iterable` is not a generator. */ - iter = PyObject_GetIter(iterable); - Py_DECREF(iterable); - SET_TOP(iter); - if (iter == NULL) - goto error; - } - PREDICT(LOAD_CONST); - DISPATCH(); - } - - case TARGET(FOR_ITER): { - PREDICTED(FOR_ITER); - /* before: [iter]; after: [iter, iter()] *or* [] */ - PyObject* iter = TOP(); - PyObject* next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next != NULL) { - PUSH(next); - PREDICT(STORE_FAST); - PREDICT(UNPACK_SEQUENCE); - DISPATCH(); - } - if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - goto error; - } else if (tstate->c_tracefunc != NULL) { - call_exc_trace( - tstate->c_tracefunc, - tstate->c_traceobj, - tstate, - f, - &trace_info); - } - _PyErr_Clear(tstate); - } - /* iterator ended normally */ - STACK_SHRINK(1); - Py_DECREF(iter); - JUMPBY(oparg); - DISPATCH(); - } - - case TARGET(SETUP_FINALLY): { - PyFrame_BlockSetup( - f, SETUP_FINALLY, INSTR_OFFSET() + oparg, STACK_LEVEL()); - DISPATCH(); - } - - case TARGET(BEFORE_ASYNC_WITH): { - _Py_IDENTIFIER(__aenter__); - _Py_IDENTIFIER(__aexit__); - PyObject* mgr = TOP(); - PyObject* enter = special_lookup(tstate, mgr, &PyId___aenter__); - PyObject* res; - if (enter == NULL) { - goto error; - } - PyObject* exit = special_lookup(tstate, mgr, &PyId___aexit__); - if (exit == NULL) { - Py_DECREF(enter); - goto error; - } - SET_TOP(exit); - Py_DECREF(mgr); - res = _PyObject_CallNoArg(enter); - Py_DECREF(enter); - if (res == NULL) - goto error; - PUSH(res); - PREDICT(GET_AWAITABLE); - DISPATCH(); - } - - case TARGET(SETUP_ASYNC_WITH): { - PyObject* res = POP(); - /* Setup the finally block before pushing the result - of __aenter__ on the stack. */ - PyFrame_BlockSetup( - f, SETUP_FINALLY, INSTR_OFFSET() + oparg, STACK_LEVEL()); - PUSH(res); - DISPATCH(); - } - - case TARGET(SETUP_WITH): { - _Py_IDENTIFIER(__enter__); - _Py_IDENTIFIER(__exit__); - PyObject* mgr = TOP(); - PyObject* enter = special_lookup(tstate, mgr, &PyId___enter__); - PyObject* res; - if (enter == NULL) { - goto error; - } - PyObject* exit = special_lookup(tstate, mgr, &PyId___exit__); - if (exit == NULL) { - Py_DECREF(enter); - goto error; - } - SET_TOP(exit); - Py_DECREF(mgr); - res = _PyObject_CallNoArg(enter); - Py_DECREF(enter); - if (res == NULL) - goto error; - /* Setup the finally block before pushing the result - of __enter__ on the stack. */ - PyFrame_BlockSetup( - f, SETUP_FINALLY, INSTR_OFFSET() + oparg, STACK_LEVEL()); - - PUSH(res); - DISPATCH(); - } - - case TARGET(WITH_EXCEPT_START): { - /* At the top of the stack are 7 values: - - (TOP, SECOND, THIRD) = exc_info() - - (FOURTH, FIFTH, SIXTH) = previous exception for EXCEPT_HANDLER - - SEVENTH: the context.__exit__ bound method - We call SEVENTH(TOP, SECOND, THIRD). - Then we push again the TOP exception and the __exit__ - return value. - */ - PyObject* exit_func; - PyObject *exc, *val, *tb, *res; - - exc = TOP(); - val = SECOND(); - tb = THIRD(); - assert(!Py_IsNone(exc)); - assert(!PyLong_Check(exc)); - exit_func = PEEK(7); - PyObject* stack[4] = {NULL, exc, val, tb}; - res = PyObject_Vectorcall( - exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - if (res == NULL) - goto error; - - PUSH(res); - DISPATCH(); - } - - case TARGET(LOAD_METHOD): { - /* Designed to work in tandem with CALL_METHOD. */ - PyObject* name = GETITEM(names, oparg); - PyObject* obj = TOP(); - PyObject* meth = NULL; - - int meth_found = shadow.shadow == NULL - ? _PyObject_GetMethod(obj, name, &meth) - : _PyShadow_LoadMethodWithCache( - &shadow, next_instr, obj, name, &meth); - - if (meth == NULL) { - /* Most likely attribute wasn't found. */ - goto error; - } - - if (meth_found) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - - meth | self | arg1 | ... | argN - */ - SET_TOP(meth); - PUSH(obj); // self - } else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL_METHOD that it's not a method call. - - NULL | meth | arg1 | ... | argN - */ - SET_TOP(NULL); - Py_DECREF(obj); - PUSH(meth); - } - DISPATCH(); - } - - case TARGET(CALL_METHOD): { - /* Designed to work in tamdem with LOAD_METHOD. */ - PyObject **sp, *res, *meth; - - sp = stack_pointer; - int awaited = IS_AWAITED(); - - meth = PEEK(oparg + 2); - if (meth == NULL) { - /* `meth` is NULL when LOAD_METHOD thinks that it's not - a method call. - - Stack layout: - - ... | NULL | callable | arg1 | ... | argN - ^- TOP() - ^- (-oparg) - ^- (-oparg-1) - ^- (-oparg-2) - - `callable` will be POPed by call_function. - NULL will will be POPed manually later. - */ - res = call_function( - tstate, - &trace_info, - &sp, - oparg, - NULL, - awaited ? Ci_Py_AWAITED_CALL_MARKER : 0); - stack_pointer = sp; - (void)POP(); /* POP the NULL. */ - } else { - /* This is a method call. Stack layout: - - ... | method | self | arg1 | ... | argN - ^- TOP() - ^- (-oparg) - ^- (-oparg-1) - ^- (-oparg-2) - - `self` and `method` will be POPed by call_function. - We'll be passing `oparg + 1` to call_function, to - make it accept the `self` as a first argument. - */ - res = call_function( - tstate, - &trace_info, - &sp, - oparg + 1, - NULL, - (awaited ? Ci_Py_AWAITED_CALL_MARKER : 0)); - stack_pointer = sp; - } - if (res == NULL) { - PUSH(NULL); - goto error; - } - if (awaited && Ci_PyWaitHandle_CheckExact(res)) { - DISPATCH_EAGER_CORO_RESULT(res, PUSH); - } - assert(!Ci_PyWaitHandle_CheckExact(res)); - PUSH(res); - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - case TARGET(CALL_FUNCTION): { - PREDICTED(CALL_FUNCTION); - PyObject **sp, *res; - sp = stack_pointer; - int awaited = IS_AWAITED(); - res = call_function( - tstate, - &trace_info, - &sp, - oparg, - NULL, - awaited ? Ci_Py_AWAITED_CALL_MARKER : 0); - stack_pointer = sp; - if (res == NULL) { - PUSH(NULL); - goto error; - } - if (awaited && Ci_PyWaitHandle_CheckExact(res)) { - DISPATCH_EAGER_CORO_RESULT(res, PUSH); - } - assert(!Ci_PyWaitHandle_CheckExact(res)); - PUSH(res); - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - case TARGET(CALL_FUNCTION_KW): { - PyObject **sp, *res, *names; - - names = POP(); - - assert(PyTuple_Check(names)); - assert(PyTuple_GET_SIZE(names) <= oparg); - /* We assume without checking that names contains only strings */ - sp = stack_pointer; - int awaited = IS_AWAITED(); - res = call_function( - tstate, - &trace_info, - &sp, - oparg, - names, - awaited ? Ci_Py_AWAITED_CALL_MARKER : 0); - stack_pointer = sp; - Py_DECREF(names); - - if (res == NULL) { - PUSH(NULL); - goto error; - } - if (awaited && Ci_PyWaitHandle_CheckExact(res)) { - DISPATCH_EAGER_CORO_RESULT(res, PUSH); - } - assert(!Ci_PyWaitHandle_CheckExact(res)); - - PUSH(res); - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - case TARGET(CALL_FUNCTION_EX): { - PREDICTED(CALL_FUNCTION_EX); - PyObject *func, *callargs, *kwargs = NULL, *result; - if (oparg & 0x01) { - kwargs = POP(); - if (!PyDict_CheckExact(kwargs)) { - PyObject* d = PyDict_New(); - if (d == NULL) - goto error; - if (_PyDict_MergeEx(d, kwargs, 2) < 0) { - Py_DECREF(d); - format_kwargs_error(tstate, SECOND(), kwargs); - Py_DECREF(kwargs); - goto error; - } - Py_DECREF(kwargs); - kwargs = d; - } - assert(PyDict_CheckExact(kwargs)); - } - callargs = POP(); - func = TOP(); - if (!PyTuple_CheckExact(callargs)) { - if (check_args_iterable(tstate, func, callargs) < 0) { - Py_DECREF(callargs); - goto error; - } - Py_SETREF(callargs, PySequence_Tuple(callargs)); - if (callargs == NULL) { - goto error; - } - } - assert(PyTuple_CheckExact(callargs)); - int awaited = IS_AWAITED(); - result = - do_call_core(tstate, &trace_info, func, callargs, kwargs, awaited); - Py_DECREF(func); - Py_DECREF(callargs); - Py_XDECREF(kwargs); - - if (result == NULL) { - SET_TOP(NULL); - goto error; - } - if (awaited && Ci_PyWaitHandle_CheckExact(result)) { - DISPATCH_EAGER_CORO_RESULT(result, SET_TOP); - } - assert(!Ci_PyWaitHandle_CheckExact(result)); - SET_TOP(result); - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - - case TARGET(MAKE_FUNCTION): { - PyObject* qualname = POP(); - PyObject* codeobj = POP(); - PyFunctionObject* func = (PyFunctionObject*)PyFunction_NewWithQualName( - codeobj, f->f_globals, qualname); - - Py_DECREF(codeobj); - Py_DECREF(qualname); - if (func == NULL) { - goto error; - } - - if (oparg & 0x08) { - assert(PyTuple_CheckExact(TOP())); - func->func_closure = POP(); - } - if (oparg & 0x04) { - assert(PyTuple_CheckExact(TOP())); - func->func_annotations = POP(); - } - if (oparg & 0x02) { - assert(PyDict_CheckExact(TOP())); - func->func_kwdefaults = POP(); - } - if (oparg & 0x01) { - assert(PyTuple_CheckExact(TOP())); - func->func_defaults = POP(); - } - - PUSH((PyObject*)func); - DISPATCH(); - } - - case TARGET(BUILD_SLICE): { - PyObject *start, *stop, *step, *slice; - if (oparg == 3) - step = POP(); - else - step = NULL; - stop = POP(); - start = TOP(); - slice = PySlice_New(start, stop, step); - Py_DECREF(start); - Py_DECREF(stop); - Py_XDECREF(step); - SET_TOP(slice); - if (slice == NULL) - goto error; - DISPATCH(); - } - - case TARGET(FORMAT_VALUE): { - /* Handles f-string value formatting. */ - PyObject* result; - PyObject* fmt_spec; - PyObject* value; - PyObject* (*conv_fn)(PyObject*); - int which_conversion = oparg & FVC_MASK; - int have_fmt_spec = (oparg & FVS_MASK) == FVS_HAVE_SPEC; - - fmt_spec = have_fmt_spec ? POP() : NULL; - value = POP(); - - /* See if any conversion is specified. */ - switch (which_conversion) { - case FVC_NONE: - conv_fn = NULL; - break; - case FVC_STR: - conv_fn = PyObject_Str; - break; - case FVC_REPR: - conv_fn = PyObject_Repr; - break; - case FVC_ASCII: - conv_fn = PyObject_ASCII; - break; - default: - _PyErr_Format( - tstate, - PyExc_SystemError, - "unexpected conversion flag %d", - which_conversion); - goto error; - } - - /* If there's a conversion function, call it and replace - value with that result. Otherwise, just use value, - without conversion. */ - if (conv_fn != NULL) { - result = conv_fn(value); - Py_DECREF(value); - if (result == NULL) { - Py_XDECREF(fmt_spec); - goto error; - } - value = result; - } - - /* If value is a unicode object, and there's no fmt_spec, - then we know the result of format(value) is value - itself. In that case, skip calling format(). I plan to - move this optimization in to PyObject_Format() - itself. */ - if (PyUnicode_CheckExact(value) && fmt_spec == NULL) { - /* Do nothing, just transfer ownership to result. */ - result = value; - } else { - /* Actually call format(). */ - result = PyObject_Format(value, fmt_spec); - Py_DECREF(value); - Py_XDECREF(fmt_spec); - if (result == NULL) { - goto error; - } - } - - PUSH(result); - DISPATCH(); - } - - case TARGET(ROT_N): { - PyObject* top = TOP(); - memmove( - &PEEK(oparg - 1), &PEEK(oparg), sizeof(PyObject*) * (oparg - 1)); - PEEK(oparg) = top; - DISPATCH(); - } - - case TARGET(SHADOW_NOP): { - DISPATCH(); - } - - case TARGET(LOAD_GLOBAL_CACHED): { - PyObject* name; - PyObject* v = *global_cache[(unsigned int)oparg]; - - if (v == NULL) { - name = _PyShadow_GetOriginalName(&shadow, next_instr); - v = Cix_PyDict_LoadGlobal( - (PyDictObject*)f->f_globals, (PyDictObject*)f->f_builtins, name); - if (v == NULL) { - if (!PyErr_Occurred()) { - /* Cix_PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ - format_exc_check_arg( - tstate, PyExc_NameError, NAME_ERROR_MSG, name); - } - goto error; - } - } - Py_INCREF(v); - PUSH(v); - DISPATCH(); - } - - case TARGET(LOAD_ATTR_NO_DICT_DESCR): { - PyObject* owner = TOP(); - _PyShadow_InstanceAttrEntry* entry = - _PyShadow_GetInstanceAttr(&shadow, oparg); - PyObject* res = - _PyShadow_LoadAttrNoDictDescr(&shadow, next_instr, entry, owner); - if (res == NULL) - goto error; - - Py_DECREF(owner); - SET_TOP(res); - DISPATCH(); - } - - case TARGET(LOAD_ATTR_DICT_DESCR): { - PyObject* owner = TOP(); - _PyShadow_InstanceAttrEntry* entry = - _PyShadow_GetInstanceAttr(&shadow, oparg); - PyObject* res = - _PyShadow_LoadAttrDictDescr(&shadow, next_instr, entry, owner); - if (res == NULL) - goto error; - - Py_DECREF(owner); - SET_TOP(res); - DISPATCH(); - } - - case TARGET(LOAD_ATTR_DICT_NO_DESCR): { - PyObject* owner = TOP(); - _PyShadow_InstanceAttrEntry* entry = - _PyShadow_GetInstanceAttr(&shadow, oparg); - PyObject* res = - _PyShadow_LoadAttrDictNoDescr(&shadow, next_instr, entry, owner); - if (res == NULL) - goto error; - - Py_DECREF(owner); - SET_TOP(res); - DISPATCH(); - } - - case TARGET(LOAD_ATTR_SLOT): { - PyObject* owner = TOP(); - _PyShadow_InstanceAttrEntry* entry = - _PyShadow_GetInstanceAttr(&shadow, oparg); - PyObject* res = - _PyShadow_LoadAttrSlot(&shadow, next_instr, entry, owner); - if (res == NULL) - goto error; - - SET_TOP(res); - Py_DECREF(owner); - DISPATCH(); - } - - case TARGET(LOAD_ATTR_SPLIT_DICT): { - PyObject* owner = TOP(); - _PyShadow_InstanceAttrEntry* entry = - _PyShadow_GetInstanceAttr(&shadow, oparg); - PyObject* res = - _PyShadow_LoadAttrSplitDict(&shadow, next_instr, entry, owner); - if (res == NULL) - goto error; - - SET_TOP(res); - Py_DECREF(owner); - DISPATCH(); - } - - case TARGET(LOAD_ATTR_SPLIT_DICT_DESCR): { - /* Normal descriptor + split dict. We're probably looking up a - * method and likely have a splitoffset of -1 */ - PyObject* owner = TOP(); - _PyShadow_InstanceAttrEntry* entry = - _PyShadow_GetInstanceAttr(&shadow, oparg); - PyObject* res = - _PyShadow_LoadAttrSplitDictDescr(&shadow, next_instr, entry, owner); - if (res == NULL) - goto error; - - Py_DECREF(owner); - SET_TOP(res); - DISPATCH(); - } - - case TARGET(LOAD_ATTR_TYPE): { - _PyShadow_InstanceAttrEntry* entry = - _PyShadow_GetInstanceAttr(&shadow, oparg); - PyObject* owner = TOP(); - PyObject* res = - _PyShadow_LoadAttrType(&shadow, next_instr, entry, owner); - if (res == NULL) - goto error; - - Py_DECREF(owner); - SET_TOP(res); - DISPATCH(); - } - - case TARGET(LOAD_ATTR_MODULE): { - PyObject* owner = TOP(); - _PyShadow_ModuleAttrEntry* entry = - _PyShadow_GetModuleAttr(&shadow, oparg); - PyObject* res = - _PyShadow_LoadAttrModule(&shadow, next_instr, entry, owner); - if (res == NULL) - goto error; - - Py_DECREF(owner); - SET_TOP(res); - DISPATCH(); - } - - case TARGET(LOAD_ATTR_S_MODULE): { - PyObject* owner = TOP(); - _PyShadow_ModuleAttrEntry* entry = - _PyShadow_GetStrictModuleAttr(&shadow, oparg); - PyObject* res = - _PyShadow_LoadAttrStrictModule(&shadow, next_instr, entry, owner); - if (res == NULL) - goto error; - - Py_DECREF(owner); - SET_TOP(res); - DISPATCH(); - } - - case TARGET(LOAD_ATTR_UNCACHABLE): { - PyObject* name = GETITEM(names, oparg); - PyObject* owner = TOP(); - INLINE_CACHE_UNCACHABLE_TYPE(Py_TYPE(owner)); - - INLINE_CACHE_RECORD_STAT(LOAD_ATTR_UNCACHABLE, hits); - PyObject* res = PyObject_GetAttr(owner, name); - Py_DECREF(owner); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(LOAD_ATTR_POLYMORPHIC): { - PyObject* owner = TOP(); - PyObject* res; - _PyShadow_InstanceAttrEntry** entries = - _PyShadow_GetPolymorphicAttr(&shadow, oparg); - PyTypeObject* type = Py_TYPE(owner); - for (int i = 0; i < POLYMORPHIC_CACHE_SIZE; i++) { - _PyShadow_InstanceAttrEntry* entry = entries[i]; - if (entry == NULL) { - continue; - } else if (entry->type != type) { - if (entry->type == NULL) { - Py_CLEAR(entries[i]); - } - continue; - } - - switch (((_PyCacheType*)Py_TYPE(entry))->load_attr_opcode) { - case LOAD_ATTR_NO_DICT_DESCR: - res = _PyShadow_LoadAttrNoDictDescrHit(entry, owner); - break; - case LOAD_ATTR_DICT_DESCR: - res = _PyShadow_LoadAttrDictDescrHit(entry, owner); - break; - case LOAD_ATTR_DICT_NO_DESCR: - res = _PyShadow_LoadAttrDictNoDescrHit(entry, owner); - break; - case LOAD_ATTR_SLOT: - res = _PyShadow_LoadAttrSlotHit(entry, owner); - break; - case LOAD_ATTR_SPLIT_DICT: - res = _PyShadow_LoadAttrSplitDictHit(entry, owner); - break; - case LOAD_ATTR_SPLIT_DICT_DESCR: - res = _PyShadow_LoadAttrSplitDictDescrHit(entry, owner); - break; - default: - Py_UNREACHABLE(); - return NULL; - } - if (res == NULL) - goto error; - - Py_DECREF(owner); - SET_TOP(res); - DISPATCH(); - } - res = - _PyShadow_LoadAttrPolymorphic(&shadow, next_instr, entries, owner); - - if (res == NULL) - goto error; - - Py_DECREF(owner); - SET_TOP(res); - DISPATCH(); - } - - case TARGET(STORE_ATTR_UNCACHABLE): { - PyObject* name = GETITEM(names, oparg); - PyObject* owner = TOP(); - PyObject* v = SECOND(); - int err; - STACK_SHRINK(2); - err = PyObject_SetAttr(owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); - if (err != 0) - goto error; - DISPATCH(); - } - - case TARGET(STORE_ATTR_DICT): { - PyObject* owner = TOP(); - PyObject* v = SECOND(); - _PyShadow_InstanceAttrEntry* entry = - _PyShadow_GetInstanceAttr(&shadow, oparg); - if (_PyShadow_StoreAttrDict(&shadow, next_instr, entry, owner, v)) { - goto error; - } - - STACK_SHRINK(2); - Py_DECREF(v); - Py_DECREF(owner); - DISPATCH(); - } - - case TARGET(STORE_ATTR_DESCR): { - PyObject* owner = TOP(); - PyObject* v = SECOND(); - _PyShadow_InstanceAttrEntry* entry = - _PyShadow_GetInstanceAttr(&shadow, oparg); - if (_PyShadow_StoreAttrDescr(&shadow, next_instr, entry, owner, v)) { - goto error; - } - - STACK_SHRINK(2); - Py_DECREF(v); - Py_DECREF(owner); - DISPATCH(); - } - - case TARGET(STORE_ATTR_SPLIT_DICT): { - PyObject* owner = TOP(); - PyObject* v = SECOND(); - _PyShadow_InstanceAttrEntry* entry = - _PyShadow_GetInstanceAttr(&shadow, oparg); - if (_PyShadow_StoreAttrSplitDict( - &shadow, next_instr, entry, owner, v)) { - goto error; - } - - STACK_SHRINK(2); - Py_DECREF(v); - Py_DECREF(owner); - DISPATCH(); - } - - case TARGET(STORE_ATTR_SLOT): { - PyObject* owner = TOP(); - PyObject* v = SECOND(); - _PyShadow_InstanceAttrEntry* entry = - _PyShadow_GetInstanceAttr(&shadow, oparg); - if (_PyShadow_StoreAttrSlot(&shadow, next_instr, entry, owner, v)) { - goto error; - } - - STACK_SHRINK(2); - Py_DECREF(v); - Py_DECREF(owner); - DISPATCH(); - } - -#define SHADOW_LOAD_METHOD(func, type, helper) \ - PyObject* obj = TOP(); \ - PyObject* meth = NULL; \ - type* entry = helper(&shadow, oparg); \ - int meth_found = func(&shadow, next_instr, entry, obj, &meth); \ - if (meth == NULL) { \ - /* Most likely attribute wasn't found. */ \ - goto error; \ - } \ - if (meth_found) { \ - SET_TOP(meth); \ - PUSH(obj); \ - } else { \ - SET_TOP(NULL); \ - Py_DECREF(obj); \ - PUSH(meth); \ - } \ - DISPATCH(); - - case TARGET(LOAD_METHOD_MODULE): { - SHADOW_LOAD_METHOD( - _PyShadow_LoadMethodModule, - _PyShadow_ModuleAttrEntry, - _PyShadow_GetModuleAttr); - } - - case TARGET(LOAD_METHOD_S_MODULE): { - SHADOW_LOAD_METHOD( - _PyShadow_LoadMethodStrictModule, - _PyShadow_ModuleAttrEntry, - _PyShadow_GetStrictModuleAttr); - } - - case TARGET(LOAD_METHOD_SPLIT_DICT_DESCR): { - SHADOW_LOAD_METHOD( - _PyShadow_LoadMethodSplitDictDescr, - _PyShadow_InstanceAttrEntry, - _PyShadow_GetInstanceAttr); - } - - case TARGET(LOAD_METHOD_DICT_DESCR): { - SHADOW_LOAD_METHOD( - _PyShadow_LoadMethodDictDescr, - _PyShadow_InstanceAttrEntry, - _PyShadow_GetInstanceAttr); - } - - case TARGET(LOAD_METHOD_NO_DICT_DESCR): { - SHADOW_LOAD_METHOD( - _PyShadow_LoadMethodNoDictDescr, - _PyShadow_InstanceAttrEntry, - _PyShadow_GetInstanceAttr); - } - - case TARGET(LOAD_METHOD_TYPE): { - SHADOW_LOAD_METHOD( - _PyShadow_LoadMethodType, - _PyShadow_InstanceAttrEntry, - _PyShadow_GetInstanceAttr); - } - - case TARGET(LOAD_METHOD_TYPE_METHODLIKE): { - SHADOW_LOAD_METHOD( - _PyShadow_LoadMethodTypeMethodLike, - _PyShadow_InstanceAttrEntry, - _PyShadow_GetInstanceAttr); - } - - case TARGET(LOAD_METHOD_DICT_METHOD): { - SHADOW_LOAD_METHOD( - _PyShadow_LoadMethodDictMethod, - _PyShadow_InstanceAttrEntry, - _PyShadow_GetInstanceAttr); - } - - case TARGET(LOAD_METHOD_SPLIT_DICT_METHOD): { - SHADOW_LOAD_METHOD( - _PyShadow_LoadMethodSplitDictMethod, - _PyShadow_InstanceAttrEntry, - _PyShadow_GetInstanceAttr); - } - - case TARGET(LOAD_METHOD_NO_DICT_METHOD): { - SHADOW_LOAD_METHOD( - _PyShadow_LoadMethodNoDictMethod, - _PyShadow_InstanceAttrEntry, - _PyShadow_GetInstanceAttr); - } - - case TARGET(LOAD_METHOD_UNSHADOWED_METHOD): { - SHADOW_LOAD_METHOD( - _PyShadow_LoadMethodUnshadowedMethod, - _PyShadow_InstanceAttrEntry, - _PyShadow_GetInstanceAttr); - } - - case TARGET(LOAD_METHOD_UNCACHABLE): { - /* Designed to work in tandem with CALL_METHOD. */ - PyObject* name = GETITEM(names, oparg); - PyObject* obj = TOP(); - PyObject* meth = NULL; - - int meth_found = _PyObject_GetMethod(obj, name, &meth); - - if (meth == NULL) { - /* Most likely attribute wasn't found. */ - goto error; - } - - if (meth_found) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - - meth | self | arg1 | ... | argN - */ - SET_TOP(meth); - PUSH(obj); // self - } else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL_METHOD that it's not a method call. - - NULL | meth | arg1 | ... | argN - */ - SET_TOP(NULL); - Py_DECREF(obj); - PUSH(meth); - } - DISPATCH(); - } - - case TARGET(BINARY_SUBSCR_TUPLE_CONST_INT): { - PyObject* container = TOP(); - PyObject* res; - PyObject* sub; - if (PyTuple_CheckExact(container)) { - Py_ssize_t i = (Py_ssize_t)oparg; - if (i < 0) { - i += PyTuple_GET_SIZE(container); - } - if (i < 0 || i >= Py_SIZE(container)) { - PyErr_SetString(PyExc_IndexError, "tuple index out of range"); - res = NULL; - } else { - res = ((PyTupleObject*)container)->ob_item[oparg]; - Py_INCREF(res); - } - } else { - sub = PyLong_FromLong(oparg); - res = PyObject_GetItem(container, sub); - Py_DECREF(sub); - } - Py_DECREF(container); - - SET_TOP(res); - if (res == NULL) - goto error; - // This shadow code is applied when we have - // LOAD_CONST i - // BINARY_SUBSCR - // And is patched into BINARY_SUBSCR_TUPLE_CONST_INT i - // at the position of LOAD_CONST. - // This means that we should always skip the next instruction - // (i.e. the BINARY_SUBSCR) - NEXTOPARG(); - - DISPATCH(); - } - case TARGET(BINARY_SUBSCR_DICT_STR): { - PyObject* sub = POP(); - PyObject* container = TOP(); - PyObject* res; - if (PyDict_CheckExact(container) && PyUnicode_CheckExact(sub)) { - res = _PyDict_GetItem_Unicode(container, sub); - if (res == NULL) { - _PyErr_SetKeyError(sub); - } else { - Py_INCREF(res); - } - } else { - _PyShadow_PatchByteCode(&shadow, next_instr, BINARY_SUBSCR, oparg); - res = PyObject_GetItem(container, sub); - } - - Py_DECREF(container); - Py_DECREF(sub); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(BINARY_SUBSCR_TUPLE): { - PyObject* sub = POP(); - PyObject* container = TOP(); - PyObject* res; - if (PyTuple_CheckExact(container)) { - res = Ci_tuple_subscript(container, sub); - } else { - _PyShadow_PatchByteCode(&shadow, next_instr, BINARY_SUBSCR, oparg); - res = PyObject_GetItem(container, sub); - } - - Py_DECREF(container); - Py_DECREF(sub); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(BINARY_SUBSCR_LIST): { - PyObject* sub = POP(); - PyObject* container = TOP(); - PyObject* res; - if (PyList_CheckExact(container)) { - res = Ci_list_subscript(container, sub); - } else { - _PyShadow_PatchByteCode(&shadow, next_instr, BINARY_SUBSCR, oparg); - res = PyObject_GetItem(container, sub); - } - - Py_DECREF(container); - Py_DECREF(sub); - SET_TOP(res); - if (res == NULL) - goto error; - DISPATCH(); - } - - case TARGET(BINARY_SUBSCR_DICT): { - PyObject* sub = POP(); - PyObject* container = TOP(); - PyObject* res; - if (PyDict_CheckExact(container)) { - res = Ci_dict_subscript(container, sub); - } else { - _PyShadow_PatchByteCode(&shadow, next_instr, BINARY_SUBSCR, oparg); - res = PyObject_GetItem(container, sub); - } - - Py_DECREF(container); - Py_DECREF(sub); - SET_TOP(res); - if (res == NULL) - goto error; - - DISPATCH(); - } - - case TARGET(EXTENDED_ARG): { - int oldoparg = oparg; - NEXTOPARG(); - oparg |= oldoparg << 8; - goto dispatch_opcode; - } - -#define _POST_INVOKE_CLEANUP_PUSH_DISPATCH(nargs, awaited, res) \ - while (nargs--) { \ - Py_DECREF(POP()); \ - } \ - if (res == NULL) { \ - goto error; \ - } \ - if (awaited && Ci_PyWaitHandle_CheckExact(res)) { \ - DISPATCH_EAGER_CORO_RESULT(res, PUSH); \ - } \ - assert(!Ci_PyWaitHandle_CheckExact(res)); \ - PUSH(res); \ - DISPATCH(); - - case TARGET(LOAD_METHOD_STATIC): { - PyObject* value = GETITEM(consts, oparg); - PyObject* target = PyTuple_GET_ITEM(value, 0); - bool is_classmethod = _PyClassLoader_IsClassMethodDescr(value); - - Py_ssize_t slot = _PyClassLoader_ResolveMethod(target); - if (slot == -1) { - goto error; - } - - assert(*(next_instr - 2) == EXTENDED_ARG); - if (shadow.shadow != NULL && slot < 0x80) { - /* We smuggle in the information about whether the invocation was a - * classmethod in the low bit of the oparg. This is necessary, as - * without, the runtime won't be able to get the correct vtable from - * self when the type is passed in. - */ - _PyShadow_PatchByteCode( - &shadow, - next_instr, - LOAD_METHOD_STATIC_CACHED, - load_method_static_cached_oparg(slot, is_classmethod)); - } - - PyObject* self = POP(); - - _PyType_VTable* vtable; - if (is_classmethod) { - vtable = (_PyType_VTable*)(((PyTypeObject*)self)->tp_cache); - } else { - vtable = (_PyType_VTable*)self->ob_type->tp_cache; - } - - assert(!PyErr_Occurred()); - StaticMethodInfo res = - _PyClassLoader_LoadStaticMethod(vtable, slot, self); - if (res.lmr_func == NULL) { - Py_DECREF(self); - goto error; - } - PUSH(res.lmr_func); - PUSH(self); - DISPATCH(); - } - - case TARGET(LOAD_METHOD_STATIC_CACHED): { - bool is_classmethod = - load_method_static_cached_oparg_is_classmethod(oparg); - PyObject* self = POP(); - - PyTypeObject* ty = is_classmethod ? (PyTypeObject*)self : Py_TYPE(self); - _PyType_VTable* vtable = (_PyType_VTable*)ty->tp_cache; - - Py_ssize_t slot = load_method_static_cached_oparg_slot(oparg); - - StaticMethodInfo res = - _PyClassLoader_LoadStaticMethod(vtable, slot, self); - assert(res.lmr_func != NULL); - assert(self != NULL); - PUSH(res.lmr_func); - PUSH(self); - DISPATCH(); - } - - case TARGET(INVOKE_METHOD): { - // This is identical to CALL_FUNCTION_EX in the interpreter except self - // isn't included in the oparg count. - PyObject* value = GETITEM(consts, oparg); - Py_ssize_t nargs = PyLong_AsLong(PyTuple_GET_ITEM(value, 1)) + 1; - PyObject **sp, *res; - sp = stack_pointer; - int awaited = IS_AWAITED(); - res = call_function( - tstate, - &trace_info, - &sp, - nargs, - NULL, - awaited ? Ci_Py_AWAITED_CALL_MARKER : 0); - stack_pointer = sp; - if (res == NULL) { - PUSH(NULL); - goto error; - } - if (awaited && Ci_PyWaitHandle_CheckExact(res)) { - DISPATCH_EAGER_CORO_RESULT(res, PUSH); - } - assert(!Ci_PyWaitHandle_CheckExact(res)); - PUSH(res); - CHECK_EVAL_BREAKER(); - DISPATCH(); - } - -#define FIELD_OFFSET(self, offset) (PyObject**)(((char*)self) + offset) - case TARGET(LOAD_FIELD): { - PyObject* field = GETITEM(consts, oparg); - int field_type; - Py_ssize_t offset = - _PyClassLoader_ResolveFieldOffset(field, &field_type); - if (offset == -1) { - goto error; - } - PyObject* self = TOP(); - PyObject* value; - if (field_type == TYPED_OBJECT) { - value = *FIELD_OFFSET(self, offset); - if (shadow.shadow != NULL) { - assert(offset % sizeof(PyObject*) == 0); - _PyShadow_PatchByteCode( - &shadow, - next_instr, - LOAD_OBJ_FIELD, - offset / sizeof(PyObject*)); - } - - if (value == NULL) { - PyObject* name = - PyTuple_GET_ITEM(field, PyTuple_GET_SIZE(field) - 1); - PyErr_Format( - PyExc_AttributeError, - "'%.50s' object has no attribute '%U'", - Py_TYPE(self)->tp_name, - name); - goto error; - } - Py_INCREF(value); - } else { - if (shadow.shadow != NULL) { - int pos = _PyShadow_CacheFieldType(&shadow, offset, field_type); - if (pos != -1) { - _PyShadow_PatchByteCode( - &shadow, next_instr, LOAD_PRIMITIVE_FIELD, pos); - } - } - - value = load_field(field_type, (char*)FIELD_OFFSET(self, offset)); - if (value == NULL) { - goto error; - } - } - Py_DECREF(self); - SET_TOP(value); - DISPATCH(); - } - - case TARGET(STORE_FIELD): { - PyObject* field = GETITEM(consts, oparg); - int field_type; - Py_ssize_t offset = - _PyClassLoader_ResolveFieldOffset(field, &field_type); - if (offset == -1) { - goto error; - } - - PyObject* self = POP(); - PyObject* value = POP(); - PyObject** addr = FIELD_OFFSET(self, offset); - - if (field_type == TYPED_OBJECT) { - Py_XDECREF(*addr); - *addr = value; - if (shadow.shadow != NULL) { - assert(offset % sizeof(PyObject*) == 0); - _PyShadow_PatchByteCode( - &shadow, - next_instr, - STORE_OBJ_FIELD, - offset / sizeof(PyObject*)); - } - } else { - if (shadow.shadow != NULL) { - int pos = _PyShadow_CacheFieldType(&shadow, offset, field_type); - if (pos != -1) { - _PyShadow_PatchByteCode( - &shadow, next_instr, STORE_PRIMITIVE_FIELD, pos); - } - } - store_field(field_type, (char*)addr, value); - } - Py_DECREF(self); - DISPATCH(); - } - -#define CAST_COERCE_OR_ERROR(val, type, exact) \ - if (type == &PyFloat_Type && PyObject_TypeCheck(val, &PyLong_Type)) { \ - long lval = PyLong_AsLong(val); \ - Py_DECREF(val); \ - SET_TOP(PyFloat_FromDouble(lval)); \ - } else { \ - PyErr_Format( \ - PyExc_TypeError, \ - exact ? "expected exactly '%s', got '%s'" : "expected '%s', got '%s'", \ - type->tp_name, \ - Py_TYPE(val)->tp_name); \ - Py_DECREF(type); \ - goto error; \ - } - - case TARGET(CAST): { - PyObject* val = TOP(); - int optional; - int exact; - PyTypeObject* type = _PyClassLoader_ResolveType( - GETITEM(consts, oparg), &optional, &exact); - if (type == NULL) { - goto error; - } - if (!_PyObject_TypeCheckOptional(val, type, optional, exact)) { - CAST_COERCE_OR_ERROR(val, type, exact); - } - - if (shadow.shadow != NULL) { - int offset = _PyShadow_CacheCastType(&shadow, (PyObject*)type); - if (offset != -1) { - if (optional) { - if (exact) { - _PyShadow_PatchByteCode( - &shadow, next_instr, CAST_CACHED_OPTIONAL_EXACT, offset); - } else { - _PyShadow_PatchByteCode( - &shadow, next_instr, CAST_CACHED_OPTIONAL, offset); - } - } else if (exact) { - _PyShadow_PatchByteCode( - &shadow, next_instr, CAST_CACHED_EXACT, offset); - } else { - _PyShadow_PatchByteCode(&shadow, next_instr, CAST_CACHED, offset); - } - } - } - Py_DECREF(type); - DISPATCH(); - } - - case TARGET(LOAD_LOCAL): { - int index = _PyLong_AsInt(PyTuple_GET_ITEM(GETITEM(consts, oparg), 0)); - - PyObject* value = GETLOCAL(index); - if (value == NULL) { - value = PyLong_FromLong(0); - SETLOCAL(index, value); /* will steal the ref */ - } - PUSH(value); - Py_INCREF(value); - - DISPATCH(); - } - - case TARGET(STORE_LOCAL): { - PyObject* local = GETITEM(consts, oparg); - int index = _PyLong_AsInt(PyTuple_GET_ITEM(local, 0)); - int type = - _PyClassLoader_ResolvePrimitiveType(PyTuple_GET_ITEM(local, 1)); - - if (type < 0) { - goto error; - } - - if (type == TYPED_DOUBLE) { - SETLOCAL(index, POP()); - } else { - Py_ssize_t val = unbox_primitive_int_and_decref(POP()); - SETLOCAL(index, box_primitive(type, val)); - } - if (shadow.shadow != NULL) { - assert(type < 8); - _PyShadow_PatchByteCode( - &shadow, next_instr, PRIMITIVE_STORE_FAST, (index << 4) | type); - } - - DISPATCH(); - } - - case TARGET(PRIMITIVE_BOX): { - if ((oparg & (TYPED_INT_SIGNED)) && oparg != (TYPED_DOUBLE)) { - /* We have a boxed value on the stack already, but we may have to - * deal with sign extension */ - PyObject* val = TOP(); - size_t ival = (size_t)PyLong_AsVoidPtr(val); - if (ival & ((size_t)1) << 63) { - SET_TOP(PyLong_FromSsize_t((int64_t)ival)); - Py_DECREF(val); - } - } - DISPATCH(); - } - - case TARGET(POP_JUMP_IF_ZERO): { - PyObject* cond = POP(); - int is_nonzero = Py_SIZE(cond); - Py_DECREF(cond); - if (!is_nonzero) { - JUMPTO(oparg); - } - DISPATCH(); - } - - case TARGET(POP_JUMP_IF_NONZERO): { - PyObject* cond = POP(); - int is_nonzero = Py_SIZE(cond); - Py_DECREF(cond); - if (is_nonzero) { - JUMPTO(oparg); - } - DISPATCH(); - } - - case TARGET(PRIMITIVE_UNBOX): { - /* We always box values in the interpreter loop, so this just does - * overflow checking here. Oparg indicates the type of the unboxed - * value. */ - PyObject* top = TOP(); - if (PyLong_CheckExact(top)) { - size_t value; - if (!_PyClassLoader_OverflowCheck(top, oparg, &value)) { - PyErr_SetString(PyExc_OverflowError, "int overflow"); - goto error; - } - } - - DISPATCH(); - } - -#define INT_BIN_OPCODE_UNSIGNED(opid, op) \ - case opid: { \ - r = POP(); \ - l = POP(); \ - PUSH(PyLong_FromVoidPtr((void*)(((size_t)PyLong_AsVoidPtr(l))op( \ - (size_t)PyLong_AsVoidPtr(r))))); \ - Py_DECREF(r); \ - Py_DECREF(l); \ - DISPATCH(); \ - } - -#define INT_BIN_OPCODE_SIGNED(opid, op) \ - case opid: { \ - r = POP(); \ - l = POP(); \ - PUSH(PyLong_FromVoidPtr((void*)(((Py_ssize_t)PyLong_AsVoidPtr(l))op( \ - (Py_ssize_t)PyLong_AsVoidPtr(r))))); \ - Py_DECREF(r); \ - Py_DECREF(l); \ - DISPATCH(); \ - } - -#define DOUBLE_BIN_OPCODE(opid, op) \ - case opid: { \ - r = POP(); \ - l = POP(); \ - PUSH( \ - (PyFloat_FromDouble((PyFloat_AS_DOUBLE(l))op(PyFloat_AS_DOUBLE(r))))); \ - Py_DECREF(r); \ - Py_DECREF(l); \ - DISPATCH(); \ - } - - case TARGET(PRIMITIVE_BINARY_OP): { - PyObject *l, *r; - switch (oparg) { - INT_BIN_OPCODE_SIGNED(PRIM_OP_ADD_INT, +) - INT_BIN_OPCODE_SIGNED(PRIM_OP_SUB_INT, -) - INT_BIN_OPCODE_SIGNED(PRIM_OP_MUL_INT, *) - INT_BIN_OPCODE_SIGNED(PRIM_OP_DIV_INT, /) - INT_BIN_OPCODE_SIGNED(PRIM_OP_MOD_INT, %) - case PRIM_OP_POW_INT: { - r = POP(); - l = POP(); - double power = - pow((Py_ssize_t)PyLong_AsVoidPtr(l), - (Py_ssize_t)PyLong_AsVoidPtr(r)); - PUSH(PyFloat_FromDouble(power)); - Py_DECREF(r); - Py_DECREF(l); - DISPATCH(); - } - case PRIM_OP_POW_UN_INT: { - r = POP(); - l = POP(); - double power = - pow((size_t)PyLong_AsVoidPtr(l), (size_t)PyLong_AsVoidPtr(r)); - PUSH(PyFloat_FromDouble(power)); - Py_DECREF(r); - Py_DECREF(l); - DISPATCH(); - } - - INT_BIN_OPCODE_SIGNED(PRIM_OP_LSHIFT_INT, <<) - INT_BIN_OPCODE_SIGNED(PRIM_OP_RSHIFT_INT, >>) - INT_BIN_OPCODE_SIGNED(PRIM_OP_XOR_INT, ^) - INT_BIN_OPCODE_SIGNED(PRIM_OP_OR_INT, |) - INT_BIN_OPCODE_SIGNED(PRIM_OP_AND_INT, &) - INT_BIN_OPCODE_UNSIGNED(PRIM_OP_MOD_UN_INT, %) - INT_BIN_OPCODE_UNSIGNED(PRIM_OP_DIV_UN_INT, /) - INT_BIN_OPCODE_UNSIGNED(PRIM_OP_RSHIFT_UN_INT, >>) - DOUBLE_BIN_OPCODE(PRIM_OP_ADD_DBL, +) - DOUBLE_BIN_OPCODE(PRIM_OP_SUB_DBL, -) - DOUBLE_BIN_OPCODE(PRIM_OP_MUL_DBL, *) - DOUBLE_BIN_OPCODE(PRIM_OP_DIV_DBL, /) - case PRIM_OP_POW_DBL: { - r = POP(); - l = POP(); - double power = pow(PyFloat_AsDouble(l), PyFloat_AsDouble(r)); - PUSH(PyFloat_FromDouble(power)); - Py_DECREF(r); - Py_DECREF(l); - DISPATCH(); - } - } - - PyErr_SetString(PyExc_RuntimeError, "unknown op"); - goto error; - } - -#define INT_UNARY_OPCODE(opid, op) \ - case opid: { \ - val = POP(); \ - PUSH(PyLong_FromVoidPtr((void*)(op(size_t) PyLong_AsVoidPtr(val)))); \ - Py_DECREF(val); \ - DISPATCH(); \ - } - -#define DBL_UNARY_OPCODE(opid, op) \ - case opid: { \ - val = POP(); \ - PUSH(PyFloat_FromDouble(op(PyFloat_AS_DOUBLE(val)))); \ - Py_DECREF(val); \ - DISPATCH(); \ - } - - case TARGET(PRIMITIVE_UNARY_OP): { - PyObject* val; - switch (oparg) { - INT_UNARY_OPCODE(PRIM_OP_NEG_INT, -) - INT_UNARY_OPCODE(PRIM_OP_INV_INT, ~) - DBL_UNARY_OPCODE(PRIM_OP_NEG_DBL, -) - case PRIM_OP_NOT_INT: { - val = POP(); - PyObject* res = PyLong_AsVoidPtr(val) ? Py_False : Py_True; - Py_INCREF(res); - PUSH(res); - Py_DECREF(val); - DISPATCH(); - } - } - PyErr_SetString(PyExc_RuntimeError, "unknown op"); - goto error; - } - -#define INT_CMP_OPCODE_UNSIGNED(opid, op) \ - case opid: { \ - r = POP(); \ - l = POP(); \ - right = (size_t)PyLong_AsVoidPtr(r); \ - left = (size_t)PyLong_AsVoidPtr(l); \ - Py_DECREF(r); \ - Py_DECREF(l); \ - res = (left op right) ? Py_True : Py_False; \ - Py_INCREF(res); \ - PUSH(res); \ - DISPATCH(); \ - } - -#define INT_CMP_OPCODE_SIGNED(opid, op) \ - case opid: { \ - r = POP(); \ - l = POP(); \ - sright = (Py_ssize_t)PyLong_AsVoidPtr(r); \ - sleft = (Py_ssize_t)PyLong_AsVoidPtr(l); \ - Py_DECREF(r); \ - Py_DECREF(l); \ - res = (sleft op sright) ? Py_True : Py_False; \ - Py_INCREF(res); \ - PUSH(res); \ - DISPATCH(); \ - } - -#define DBL_CMP_OPCODE(opid, op) \ - case opid: { \ - r = POP(); \ - l = POP(); \ - res = \ - ((PyFloat_AS_DOUBLE(l) op PyFloat_AS_DOUBLE(r)) ? Py_True : Py_False); \ - Py_DECREF(r); \ - Py_DECREF(l); \ - Py_INCREF(res); \ - PUSH(res); \ - DISPATCH(); \ - } - - case TARGET(PRIMITIVE_COMPARE_OP): { - PyObject *l, *r, *res; - Py_ssize_t sleft, sright; - size_t left, right; - switch (oparg) { - INT_CMP_OPCODE_SIGNED(PRIM_OP_EQ_INT, ==) - INT_CMP_OPCODE_SIGNED(PRIM_OP_NE_INT, !=) - INT_CMP_OPCODE_SIGNED(PRIM_OP_LT_INT, <) - INT_CMP_OPCODE_SIGNED(PRIM_OP_GT_INT, >) - INT_CMP_OPCODE_SIGNED(PRIM_OP_LE_INT, <=) - INT_CMP_OPCODE_SIGNED(PRIM_OP_GE_INT, >=) - INT_CMP_OPCODE_UNSIGNED(PRIM_OP_LT_UN_INT, <) - INT_CMP_OPCODE_UNSIGNED(PRIM_OP_GT_UN_INT, >) - INT_CMP_OPCODE_UNSIGNED(PRIM_OP_LE_UN_INT, <=) - INT_CMP_OPCODE_UNSIGNED(PRIM_OP_GE_UN_INT, >=) - DBL_CMP_OPCODE(PRIM_OP_EQ_DBL, ==) - DBL_CMP_OPCODE(PRIM_OP_NE_DBL, !=) - DBL_CMP_OPCODE(PRIM_OP_LT_DBL, <) - DBL_CMP_OPCODE(PRIM_OP_GT_DBL, >) - DBL_CMP_OPCODE(PRIM_OP_LE_DBL, <=) - DBL_CMP_OPCODE(PRIM_OP_GE_DBL, >=) - } - PyErr_SetString(PyExc_RuntimeError, "unknown op"); - goto error; - } - - case TARGET(LOAD_ITERABLE_ARG): { - // TODO: Revisit this opcode, and perhaps get it to load all - // elements of an iterable to a stack. That'll help with the - // compiled code size. - PyObject* tup = POP(); - int idx = oparg; - if (!PyTuple_CheckExact(tup)) { - if (tup->ob_type->tp_iter == NULL && !PySequence_Check(tup)) { - PyErr_Format( - PyExc_TypeError, - "argument after * " - "must be an iterable, not %.200s", - tup->ob_type->tp_name); - Py_DECREF(tup); - goto error; - } - Py_SETREF(tup, PySequence_Tuple(tup)); - if (tup == NULL) { - goto error; - } - } - PyObject* element = PyTuple_GetItem(tup, idx); - if (!element) { - Py_DECREF(tup); - goto error; - } - Py_INCREF(element); - PUSH(element); - PUSH(tup); - DISPATCH(); - } - - case TARGET(LOAD_MAPPING_ARG): { - PyObject* name = POP(); - PyObject* mapping = POP(); - - if (!PyDict_Check(mapping) && !Ci_CheckedDict_Check(mapping)) { - PyErr_Format( - PyExc_TypeError, - "argument after ** " - "must be a dict, not %.200s", - mapping->ob_type->tp_name); - Py_DECREF(name); - Py_DECREF(mapping); - goto error; - } - - PyObject* value = PyDict_GetItemWithError(mapping, name); - if (value == NULL) { - if (_PyErr_Occurred(tstate)) { - Py_DECREF(name); - Py_DECREF(mapping); - goto error; - } else if (oparg == 2) { - PyErr_Format(PyExc_TypeError, "missing argument %U", name); - goto error; - } else { - /* Default value is on the stack */ - Py_DECREF(name); - Py_DECREF(mapping); - DISPATCH(); - } - } else if (oparg == 3) { - /* Remove default value */ - Py_DECREF(POP()); - } - Py_XINCREF(value); - Py_DECREF(name); - Py_DECREF(mapping); - PUSH(value); - DISPATCH(); - } - case TARGET(INVOKE_FUNCTION): { - PyObject* value = GETITEM(consts, oparg); - Py_ssize_t nargs = PyLong_AsLong(PyTuple_GET_ITEM(value, 1)); - PyObject* target = PyTuple_GET_ITEM(value, 0); - PyObject* container; - PyObject* func = _PyClassLoader_ResolveFunction(target, &container); - if (func == NULL) { - goto error; - } - int awaited = IS_AWAITED(); - PyObject** sp = stack_pointer - nargs; - PyObject* res = invoke_static_function(func, sp, nargs, awaited); - - if (shadow.shadow != NULL && nargs < 0x80) { - if (_PyClassLoader_IsImmutable(container)) { - /* frozen type, we don't need to worry about indirecting */ - int offset = _PyShadow_CacheCastType(&shadow, func); - if (offset != -1) { - _PyShadow_PatchByteCode( - &shadow, - next_instr, - INVOKE_FUNCTION_CACHED, - (nargs << 8) | offset); - } - } else { - PyObject** funcptr = _PyClassLoader_ResolveIndirectPtr(target); - int offset = _PyShadow_CacheFunction(&shadow, funcptr); - if (offset != -1) { - _PyShadow_PatchByteCode( - &shadow, - next_instr, - INVOKE_FUNCTION_INDIRECT_CACHED, - (nargs << 8) | offset); - } - } - } - - Py_DECREF(func); - Py_DECREF(container); - - _POST_INVOKE_CLEANUP_PUSH_DISPATCH(nargs, awaited, res); - } - - case TARGET(INVOKE_NATIVE): { - PyObject* value = GETITEM(consts, oparg); - assert(PyTuple_CheckExact(value)); - PyObject* target = PyTuple_GET_ITEM(value, 0); - PyObject* name = PyTuple_GET_ITEM(target, 0); - PyObject* symbol = PyTuple_GET_ITEM(target, 1); - PyObject* signature = PyTuple_GET_ITEM(value, 1); - Py_ssize_t nargs = PyTuple_GET_SIZE(signature) - 1; - PyObject** sp = stack_pointer - nargs; - PyObject* res = _PyClassloader_InvokeNativeFunction( - name, symbol, signature, sp, nargs); - _POST_INVOKE_CLEANUP_PUSH_DISPATCH(nargs, 0, res); - } - - case TARGET(JUMP_IF_ZERO_OR_POP): { - PyObject* cond = TOP(); - int is_nonzero = Py_SIZE(cond); - if (is_nonzero) { - STACK_SHRINK(1); - Py_DECREF(cond); - } else { - JUMPTO(oparg); - } - DISPATCH(); - } - - case TARGET(JUMP_IF_NONZERO_OR_POP): { - PyObject* cond = TOP(); - int is_nonzero = Py_SIZE(cond); - if (!is_nonzero) { - STACK_SHRINK(1); - Py_DECREF(cond); - } else { - JUMPTO(oparg); - } - DISPATCH() - } - - case TARGET(FAST_LEN): { - PyObject *collection = POP(), *length = NULL; - int inexact = oparg & FAST_LEN_INEXACT; - oparg &= ~FAST_LEN_INEXACT; - assert(FAST_LEN_LIST <= oparg && oparg <= FAST_LEN_STR); - if (inexact) { - if ((oparg == FAST_LEN_LIST && PyList_CheckExact(collection)) || - (oparg == FAST_LEN_DICT && PyDict_CheckExact(collection)) || - (oparg == FAST_LEN_SET && PyAnySet_CheckExact(collection)) || - (oparg == FAST_LEN_TUPLE && PyTuple_CheckExact(collection)) || - (oparg == FAST_LEN_ARRAY && - PyStaticArray_CheckExact(collection)) || - (oparg == FAST_LEN_STR && PyUnicode_CheckExact(collection))) { - inexact = 0; - } - } - if (inexact) { - Py_ssize_t res = PyObject_Size(collection); - if (res >= 0) { - length = PyLong_FromSsize_t(res); - } - } else if (oparg == FAST_LEN_DICT) { - length = PyLong_FromLong(((PyDictObject*)collection)->ma_used); - } else if (oparg == FAST_LEN_SET) { - length = PyLong_FromLong(((PySetObject*)collection)->used); - } else { - // lists, tuples, arrays are all PyVarObject and use ob_size - length = PyLong_FromLong(Py_SIZE(collection)); - } - Py_DECREF(collection); - if (length == NULL) { - goto error; - } - PUSH(length); - DISPATCH(); - } - - case TARGET(CONVERT_PRIMITIVE): { - Py_ssize_t from_type = oparg & 0xFF; - Py_ssize_t to_type = oparg >> 4; - Py_ssize_t extend_sign = - (from_type & TYPED_INT_SIGNED) && (to_type & TYPED_INT_SIGNED); - int size = to_type >> 1; - PyObject* val = TOP(); - size_t ival = (size_t)PyLong_AsVoidPtr(val); - - ival &= trunc_masks[size]; - - // Extend the sign if needed - if (extend_sign != 0 && (ival & signed_bits[size])) { - ival |= (signex_masks[size]); - } - - Py_DECREF(val); - SET_TOP(PyLong_FromSize_t(ival)); - DISPATCH(); - } - - case TARGET(LOAD_CLASS): { - PyObject* type_descr = GETITEM(consts, oparg); - int optional; - int exact; - PyTypeObject* type = - _PyClassLoader_ResolveType(type_descr, &optional, &exact); - if (type == NULL) { - goto error; - } - PUSH((PyObject*)type); - DISPATCH(); - } - - case TARGET(BUILD_CHECKED_MAP): { - PyObject* map_info = GETITEM(consts, oparg); - PyObject* map_type = PyTuple_GET_ITEM(map_info, 0); - Py_ssize_t map_size = PyLong_AsLong(PyTuple_GET_ITEM(map_info, 1)); - - int optional; - int exact; - PyTypeObject* type = - _PyClassLoader_ResolveType(map_type, &optional, &exact); - assert(!optional); - - if (shadow.shadow != NULL) { - PyObject* cache = PyTuple_New(2); - if (cache == NULL) { - goto error; - } - PyTuple_SET_ITEM(cache, 0, (PyObject*)type); - Py_INCREF(type); - PyObject* size = PyLong_FromLong(map_size); - if (size == NULL) { - Py_DECREF(cache); - goto error; - } - PyTuple_SET_ITEM(cache, 1, size); - - int offset = _PyShadow_CacheCastType(&shadow, cache); - Py_DECREF(cache); - if (offset != -1) { - _PyShadow_PatchByteCode( - &shadow, next_instr, BUILD_CHECKED_MAP_CACHED, offset); - } - } - - PyObject* map = Ci_CheckedDict_NewPresized(type, map_size); - if (map == NULL) { - goto error; - } - Py_DECREF(type); - - Ci_BUILD_DICT(map_size, Ci_CheckedDict_SetItem); - DISPATCH(); - } - - case TARGET(SEQUENCE_GET): { - PyObject *idx = POP(), *sequence, *item; - - Py_ssize_t val = (Py_ssize_t)PyLong_AsVoidPtr(idx); - - if (val == -1 && _PyErr_Occurred(tstate)) { - Py_DECREF(idx); - goto error; - } - - sequence = POP(); - - // Adjust index - if (val < 0) { - val += Py_SIZE(sequence); - } - - oparg &= ~SEQ_SUBSCR_UNCHECKED; - - if (oparg == SEQ_LIST) { - item = PyList_GetItem(sequence, val); - Py_DECREF(sequence); - if (item == NULL) { - Py_DECREF(idx); - goto error; - } - Py_INCREF(item); - } else if (oparg == SEQ_LIST_INEXACT) { - if (PyList_CheckExact(sequence) || - Py_TYPE(sequence)->tp_as_sequence->sq_item == - PyList_Type.tp_as_sequence->sq_item) { - item = PyList_GetItem(sequence, val); - Py_DECREF(sequence); - if (item == NULL) { - Py_DECREF(idx); - goto error; - } - Py_INCREF(item); - } else { - item = PyObject_GetItem(sequence, idx); - Py_DECREF(sequence); - if (item == NULL) { - Py_DECREF(idx); - goto error; - } - } - } else if (oparg == SEQ_CHECKED_LIST) { - item = Ci_CheckedList_GetItem(sequence, val); - Py_DECREF(sequence); - if (item == NULL) { - Py_DECREF(idx); - goto error; - } - } else if (oparg == SEQ_ARRAY_INT64) { - item = _Ci_StaticArray_Get(sequence, val); - Py_DECREF(sequence); - if (item == NULL) { - Py_DECREF(idx); - goto error; - } - } else { - PyErr_Format( - PyExc_SystemError, "bad oparg for SEQUENCE_GET: %d", oparg); - Py_DECREF(idx); - goto error; - } - - Py_DECREF(idx); - PUSH(item); - DISPATCH(); - } - - case TARGET(SEQUENCE_SET): { - PyObject* subscr = TOP(); - PyObject* sequence = SECOND(); - PyObject* v = THIRD(); - int err; - STACK_SHRINK(3); - - Py_ssize_t idx = (Py_ssize_t)PyLong_AsVoidPtr(subscr); - Py_DECREF(subscr); - - if (idx == -1 && _PyErr_Occurred(tstate)) { - Py_DECREF(v); - Py_DECREF(sequence); - goto error; - } - - // Adjust index - if (idx < 0) { - idx += Py_SIZE(sequence); - } - - if (oparg == SEQ_LIST) { - err = PyList_SetItem(sequence, idx, v); - - Py_DECREF(sequence); - if (err != 0) { - Py_DECREF(v); - goto error; - } - } else if (oparg == SEQ_LIST_INEXACT) { - if (PyList_CheckExact(sequence) || - Py_TYPE(sequence)->tp_as_sequence->sq_ass_item == - PyList_Type.tp_as_sequence->sq_ass_item) { - err = PyList_SetItem(sequence, idx, v); - - Py_DECREF(sequence); - if (err != 0) { - Py_DECREF(v); - goto error; - } - } else { - err = PyObject_SetItem(sequence, subscr, v); - Py_DECREF(v); - Py_DECREF(sequence); - if (err != 0) { - goto error; - } - } - } else if (oparg == SEQ_ARRAY_INT64) { - err = _Ci_StaticArray_Set(sequence, idx, v); - - Py_DECREF(sequence); - Py_DECREF(v); - if (err != 0) { - goto error; - } - } else { - PyErr_Format( - PyExc_SystemError, "bad oparg for SEQUENCE_SET: %d", oparg); - goto error; - } - DISPATCH(); - } - - case TARGET(LIST_DEL): { - PyObject* subscr = TOP(); - PyObject* list = SECOND(); - int err; - STACK_SHRINK(2); - - Py_ssize_t idx = PyLong_AsLong(subscr); - Py_DECREF(subscr); - - if (idx == -1 && _PyErr_Occurred(tstate)) { - Py_DECREF(list); - goto error; - } - - err = PyList_SetSlice(list, idx, idx + 1, NULL); - - Py_DECREF(list); - if (err != 0) { - goto error; - } - DISPATCH(); - } - - case TARGET(REFINE_TYPE): { - DISPATCH(); - } - - case TARGET(PRIMITIVE_LOAD_CONST): { - PyObject* val = PyTuple_GET_ITEM(GETITEM(consts, oparg), 0); - Py_INCREF(val); - PUSH(val); - DISPATCH(); - } - - case TARGET(RETURN_PRIMITIVE): { - retval = POP(); - - /* In the interpreter, we always return a boxed int. We have a boxed - * value on the stack already, but we may have to deal with sign - * extension. */ - if (oparg & TYPED_INT_SIGNED && oparg != TYPED_DOUBLE) { - size_t ival = (size_t)PyLong_AsVoidPtr(retval); - if (ival & ((size_t)1) << 63) { - Py_DECREF(retval); - retval = PyLong_FromSsize_t((int64_t)ival); - } - } - - assert(f->f_iblock == 0); - goto exiting; - } - - case TARGET(LOAD_METHOD_SUPER): { - PyObject* pair = GETITEM(consts, oparg); - PyObject* name_obj = PyTuple_GET_ITEM(pair, 0); - int name_idx = _PyLong_AsInt(name_obj); - PyObject* name = GETITEM(names, name_idx); - - assert(PyBool_Check(PyTuple_GET_ITEM(pair, 1))); - int call_no_args = PyTuple_GET_ITEM(pair, 1) == Py_True; - - PyObject* self = POP(); - PyObject* type = POP(); - PyObject* global_super = POP(); - - int meth_found = 0; - PyObject* attr = super_lookup_method_or_attr( - tstate, - global_super, - (PyTypeObject*)type, - self, - name, - call_no_args, - &meth_found); - Py_DECREF(type); - Py_DECREF(global_super); - - if (attr == NULL) { - Py_DECREF(self); - goto error; - } - if (meth_found) { - PUSH(attr); - PUSH(self); - } else { - Py_DECREF(self); - - PUSH(NULL); - PUSH(attr); - } - DISPATCH(); - } - - case TARGET(LOAD_ATTR_SUPER): { - PyObject* pair = GETITEM(consts, oparg); - PyObject* name_obj = PyTuple_GET_ITEM(pair, 0); - int name_idx = _PyLong_AsInt(name_obj); - PyObject* name = GETITEM(names, name_idx); - - assert(PyBool_Check(PyTuple_GET_ITEM(pair, 1))); - - int call_no_args = PyTuple_GET_ITEM(pair, 1) == Py_True; - - PyObject* self = POP(); - PyObject* type = POP(); - PyObject* global_super = POP(); - PyObject* attr = super_lookup_method_or_attr( - tstate, - global_super, - (PyTypeObject*)type, - self, - name, - call_no_args, - NULL); - Py_DECREF(type); - Py_DECREF(self); - Py_DECREF(global_super); - - if (attr == NULL) { - goto error; - } - PUSH(attr); - DISPATCH(); - } - - case TARGET(TP_ALLOC): { - int optional; - int exact; - PyTypeObject* type = _PyClassLoader_ResolveType( - GETITEM(consts, oparg), &optional, &exact); - assert(!optional); - if (type == NULL) { - goto error; - } - - PyObject* inst = type->tp_alloc(type, 0); - if (inst == NULL) { - Py_DECREF(type); - goto error; - } - PUSH(inst); - - if (shadow.shadow != NULL) { - int offset = _PyShadow_CacheCastType(&shadow, (PyObject*)type); - if (offset != -1) { - _PyShadow_PatchByteCode( - &shadow, next_instr, TP_ALLOC_CACHED, offset); - } - } - Py_DECREF(type); - DISPATCH(); - } - - case TARGET(BUILD_CHECKED_LIST): { - PyObject* list_info = GETITEM(consts, oparg); - PyObject* list_type = PyTuple_GET_ITEM(list_info, 0); - Py_ssize_t list_size = PyLong_AsLong(PyTuple_GET_ITEM(list_info, 1)); - - int optional; - int exact; - PyTypeObject* type = - _PyClassLoader_ResolveType(list_type, &optional, &exact); - assert(!optional); - - if (shadow.shadow != NULL) { - PyObject* cache = PyTuple_New(2); - if (cache == NULL) { - goto error; - } - PyTuple_SET_ITEM(cache, 0, (PyObject*)type); - Py_INCREF(type); - PyObject* size = PyLong_FromLong(list_size); - if (size == NULL) { - Py_DECREF(cache); - goto error; - } - PyTuple_SET_ITEM(cache, 1, size); - - int offset = _PyShadow_CacheCastType(&shadow, cache); - Py_DECREF(cache); - if (offset != -1) { - _PyShadow_PatchByteCode( - &shadow, next_instr, BUILD_CHECKED_LIST_CACHED, offset); - } - } - - PyObject* list = Ci_CheckedList_New(type, list_size); - if (list == NULL) { - goto error; - } - Py_DECREF(type); - - while (--list_size >= 0) { - PyObject* item = POP(); - Ci_ListOrCheckedList_SET_ITEM(list, list_size, item); - } - PUSH(list); - DISPATCH(); - } - - case TARGET(LOAD_TYPE): { - PyObject* instance = TOP(); - Py_INCREF(Py_TYPE(instance)); - SET_TOP((PyObject*)Py_TYPE(instance)); - Py_DECREF(instance); - DISPATCH(); - } - - case TARGET(BUILD_CHECKED_LIST_CACHED): { - PyObject* cache = _PyShadow_GetCastType(&shadow, oparg); - PyTypeObject* type = (PyTypeObject*)PyTuple_GET_ITEM(cache, 0); - Py_ssize_t list_size = PyLong_AsLong(PyTuple_GET_ITEM(cache, 1)); - - PyObject* list = Ci_CheckedList_New(type, list_size); - if (list == NULL) { - goto error; - } - - while (--list_size >= 0) { - PyObject* item = POP(); - PyList_SET_ITEM(list, list_size, item); - } - PUSH(list); - DISPATCH(); - } - - case TARGET(TP_ALLOC_CACHED): { - PyTypeObject* type = - (PyTypeObject*)_PyShadow_GetCastType(&shadow, oparg); - PyObject* inst = type->tp_alloc(type, 0); - if (inst == NULL) { - goto error; - } - - PUSH(inst); - DISPATCH(); - } - - case TARGET(INVOKE_FUNCTION_CACHED): { - PyObject* func = _PyShadow_GetCastType(&shadow, oparg & 0xff); - Py_ssize_t nargs = oparg >> 8; - int awaited = IS_AWAITED(); - - PyObject** sp = stack_pointer - nargs; - PyObject* res = invoke_static_function(func, sp, nargs, awaited); - - _POST_INVOKE_CLEANUP_PUSH_DISPATCH(nargs, awaited, res); - } - - case TARGET(INVOKE_FUNCTION_INDIRECT_CACHED): { - PyObject** funcref = _PyShadow_GetFunction(&shadow, oparg & 0xff); - Py_ssize_t nargs = oparg >> 8; - int awaited = IS_AWAITED(); - - PyObject** sp = stack_pointer - nargs; - PyObject* func = *funcref; - PyObject* res; - /* For indirect calls we just use _PyObject_Vectorcall, which will - * handle non-vector call objects as well. We expect in high-perf - * situations to either have frozen types or frozen strict modules */ - if (func == NULL) { - PyObject* target = PyTuple_GET_ITEM( - _PyShadow_GetOriginalConst(&shadow, next_instr), 0); - func = _PyClassLoader_ResolveFunction(target, NULL); - if (func == NULL) { - goto error; - } - - res = _PyObject_VectorcallTstate( - tstate, - func, - sp, - (awaited ? Ci_Py_AWAITED_CALL_MARKER : 0) | nargs, - NULL); - Py_DECREF(func); - } else { - res = _PyObject_VectorcallTstate( - tstate, - func, - sp, - (awaited ? Ci_Py_AWAITED_CALL_MARKER : 0) | nargs, - NULL); - } - - _POST_INVOKE_CLEANUP_PUSH_DISPATCH(nargs, awaited, res); - } - - case TARGET(BUILD_CHECKED_MAP_CACHED): { - PyObject* cache = _PyShadow_GetCastType(&shadow, oparg); - PyTypeObject* type = (PyTypeObject*)PyTuple_GET_ITEM(cache, 0); - Py_ssize_t map_size = PyLong_AsLong(PyTuple_GET_ITEM(cache, 1)); - - PyObject* map = Ci_CheckedDict_NewPresized(type, map_size); - if (map == NULL) { - goto error; - } - - Ci_BUILD_DICT(map_size, Ci_CheckedDict_SetItem); - DISPATCH(); - } - - case TARGET(PRIMITIVE_STORE_FAST): { - int type = oparg & 0xF; - int idx = oparg >> 4; - PyObject* value = POP(); - if (type == TYPED_DOUBLE) { - SETLOCAL(idx, POP()); - } else { - Py_ssize_t val = unbox_primitive_int_and_decref(value); - SETLOCAL(idx, box_primitive(type, val)); - } - - DISPATCH(); - } - - case TARGET(CAST_CACHED_OPTIONAL): { - PyObject* val = TOP(); - PyTypeObject* type = - (PyTypeObject*)_PyShadow_GetCastType(&shadow, oparg); - if (!_PyObject_TypeCheckOptional( - val, type, /* opt */ 1, /* exact */ 0)) { - CAST_COERCE_OR_ERROR(val, type, /* exact */ 0); - } - DISPATCH(); - } - - case TARGET(CAST_CACHED): { - PyObject* val = TOP(); - PyTypeObject* type = - (PyTypeObject*)_PyShadow_GetCastType(&shadow, oparg); - if (!PyObject_TypeCheck(val, type)) { - CAST_COERCE_OR_ERROR(val, type, /* exact */ 0); - } - DISPATCH(); - } - - case TARGET(CAST_CACHED_EXACT): { - PyObject* val = TOP(); - PyTypeObject* type = - (PyTypeObject*)_PyShadow_GetCastType(&shadow, oparg); - if (Py_TYPE(val) != type) { - CAST_COERCE_OR_ERROR(val, type, /* exact */ 1); - } - DISPATCH(); - } - - case TARGET(CAST_CACHED_OPTIONAL_EXACT): { - PyObject* val = TOP(); - PyTypeObject* type = - (PyTypeObject*)_PyShadow_GetCastType(&shadow, oparg); - if (!_PyObject_TypeCheckOptional( - val, type, /* opt */ 1, /* exact */ 1)) { - CAST_COERCE_OR_ERROR(val, type, /* exact */ 1); - } - DISPATCH(); - } - - case TARGET(LOAD_PRIMITIVE_FIELD): { - _FieldCache* cache = _PyShadow_GetFieldCache(&shadow, oparg); - PyObject* value = - load_field(cache->type, ((char*)TOP()) + cache->offset); - if (value == NULL) { - goto error; - } - - Py_DECREF(TOP()); - SET_TOP(value); - DISPATCH(); - } - - case TARGET(STORE_PRIMITIVE_FIELD): { - _FieldCache* cache = _PyShadow_GetFieldCache(&shadow, oparg); - PyObject* self = POP(); - PyObject* value = POP(); - store_field(cache->type, ((char*)self) + cache->offset, value); - Py_DECREF(self); - DISPATCH(); - } - - case TARGET(LOAD_OBJ_FIELD): { - PyObject* self = TOP(); - PyObject** addr = FIELD_OFFSET(self, oparg * sizeof(PyObject*)); - PyObject* value = *addr; - if (value == NULL) { - PyErr_Format( - PyExc_AttributeError, - "'%.50s' object has no attribute", - Py_TYPE(self)->tp_name); - goto error; - } - - Py_INCREF(value); - Py_DECREF(self); - SET_TOP(value); - DISPATCH(); - } - - case TARGET(STORE_OBJ_FIELD): { - Py_ssize_t offset = oparg * sizeof(PyObject*); - PyObject* self = POP(); - PyObject* value = POP(); - PyObject** addr = FIELD_OFFSET(self, offset); - Py_XDECREF(*addr); - *addr = value; - Py_DECREF(self); - DISPATCH(); - } - - case TARGET(INVOKE_METHOD_CACHED): { - int is_classmethod = oparg & 1; - Py_ssize_t nargs = (oparg >> 1) & 0xff; - PyObject** stack = stack_pointer - nargs; - PyObject* self = *stack; - _PyType_VTable* vtable; - if (is_classmethod) { - vtable = (_PyType_VTable*)(((PyTypeObject*)self)->tp_cache); - } else { - vtable = (_PyType_VTable*)self->ob_type->tp_cache; - } - - Py_ssize_t slot = oparg >> 9; - - int awaited = IS_AWAITED(); - - assert(!PyErr_Occurred()); - PyObject* res = _PyClassLoader_InvokeMethod( - vtable, - slot, - stack, - nargs | (awaited ? Ci_Py_AWAITED_CALL_MARKER : 0)); - - _POST_INVOKE_CLEANUP_PUSH_DISPATCH(nargs, awaited, res); - } - -#if USE_COMPUTED_GOTOS - _unknown_opcode: -#endif - default: - fprintf( - stderr, - "XXX lineno: %d, opcode: %d\n", - PyFrame_GetLineNumber(f), - opcode); - _PyErr_SetString(tstate, PyExc_SystemError, "unknown opcode"); - goto error; - - } /* switch */ - - /* This should never be reached. Every opcode should end with DISPATCH() - or goto error. */ - Py_UNREACHABLE(); - - error: - /* Double-check exception status. */ -#ifdef NDEBUG - if (!_PyErr_Occurred(tstate)) { - _PyErr_SetString( - tstate, PyExc_SystemError, "error return without exception set"); - } -#else - assert(_PyErr_Occurred(tstate)); -#endif - - /* Log traceback info. */ - PyTraceBack_Here(f); - - if (tstate->c_tracefunc != NULL) { - /* Make sure state is set to FRAME_EXECUTING for tracing */ - assert(f->f_state == FRAME_EXECUTING); - f->f_state = FRAME_UNWINDING; - call_exc_trace( - tstate->c_tracefunc, tstate->c_traceobj, tstate, f, &trace_info); - } - exception_unwind: - f->f_state = FRAME_UNWINDING; - /* Unwind stacks if an exception occurred */ - while (f->f_iblock > 0) { - /* Pop the current block. */ - PyTryBlock* b = &f->f_blockstack[--f->f_iblock]; - - if (b->b_type == EXCEPT_HANDLER) { - UNWIND_EXCEPT_HANDLER(b); - continue; - } - UNWIND_BLOCK(b); - if (b->b_type == SETUP_FINALLY) { - PyObject *exc, *val, *tb; - int handler = b->b_handler; - _PyErr_StackItem* exc_info = tstate->exc_info; - /* Beware, this invalidates all b->b_* fields */ - PyFrame_BlockSetup(f, EXCEPT_HANDLER, f->f_lasti, STACK_LEVEL()); - PUSH(exc_info->exc_traceback); - PUSH(exc_info->exc_value); - if (exc_info->exc_type != NULL) { - PUSH(exc_info->exc_type); - } else { - Py_INCREF(Py_None); - PUSH(Py_None); - } - _PyErr_Fetch(tstate, &exc, &val, &tb); - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - _PyErr_NormalizeException(tstate, &exc, &val, &tb); - if (tb != NULL) - PyException_SetTraceback(val, tb); - else - PyException_SetTraceback(val, Py_None); - Py_INCREF(exc); - exc_info->exc_type = exc; - Py_INCREF(val); - exc_info->exc_value = val; - exc_info->exc_traceback = tb; - if (tb == NULL) - tb = Py_None; - Py_INCREF(tb); - PUSH(tb); - PUSH(val); - PUSH(exc); - JUMPTO(handler); - /* Resume normal execution */ - f->f_state = FRAME_EXECUTING; - goto main_loop; - } - } /* unwind stack */ - - /* End the loop as we still have an error */ - break; - } /* main loop */ - - assert(retval == NULL); - assert(_PyErr_Occurred(tstate)); - - /* Pop remaining stack entries. */ - while (!EMPTY()) { - PyObject* o = POP(); - Py_XDECREF(o); - } - f->f_stackdepth = 0; - f->f_state = FRAME_RAISED; -exiting: - if (trace_info.cframe.use_tracing) { - if (tstate->c_tracefunc) { - if (call_trace_protected( - tstate->c_tracefunc, - tstate->c_traceobj, - tstate, - f, - &trace_info, - PyTrace_RETURN, - retval)) { - Py_CLEAR(retval); - } - } - if (tstate->c_profilefunc) { - if (call_trace_protected( - tstate->c_profilefunc, - tstate->c_profileobj, - tstate, - f, - &trace_info, - PyTrace_RETURN, - retval)) { - Py_CLEAR(retval); - } - } - } - - /* pop frame */ -exit_eval_frame: - /* Restore previous cframe */ - tstate->cframe = trace_info.cframe.previous; - tstate->cframe->use_tracing = trace_info.cframe.use_tracing; - - if (f->f_gen == NULL) { - _PyShadowFrame_Pop(tstate, &shadow_frame); - } - - if (PyDTrace_FUNCTION_RETURN_ENABLED()) - dtrace_function_return(f); - _Py_LeaveRecursiveCall(tstate); - tstate->frame = f->f_back; - co->co_mutable->curcalls--; - - return _Py_CheckFunctionResult(tstate, NULL, retval, __func__); -} - -static int -_Ci_CheckArgs(PyThreadState* tstate, PyFrameObject* f, PyCodeObject* co) { - // In the future we can use co_extra to store the cached arg info - PyObject** freevars = (f->f_localsplus + f->f_code->co_nlocals); - PyObject** fastlocals = f->f_localsplus; - if (co->co_mutable->shadow == NULL) { - // This funciton hasn't been optimized yet, we'll do it the slow way. - PyObject* checks = _PyClassLoader_GetCodeArgumentTypeDescrs(co); - PyObject* local; - PyObject* type_descr; - PyTypeObject* type; - int optional; - int exact; - for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(checks); i += 2) { - local = PyTuple_GET_ITEM(checks, i); - type_descr = PyTuple_GET_ITEM(checks, i + 1); - long idx = PyLong_AsLong(local); - PyObject* val; - // Look in freevars if necessary - if (idx < 0) { - assert(!_PyErr_Occurred(tstate)); - val = PyCell_GET(freevars[-(idx + 1)]); - } else { - val = fastlocals[idx]; - } - - type = _PyClassLoader_ResolveType(type_descr, &optional, &exact); - if (type == NULL) { - return -1; - } - - int primitive = _PyClassLoader_GetTypeCode(type); - if (primitive == TYPED_BOOL) { - optional = 0; - Py_DECREF(type); - type = &PyBool_Type; - Py_INCREF(type); - } else if (primitive <= TYPED_INT64) { - exact = optional = 0; - Py_DECREF(type); - type = &PyLong_Type; - Py_INCREF(type); - } else if (primitive == TYPED_DOUBLE) { - exact = optional = 0; - Py_DECREF(type); - type = &PyFloat_Type; - Py_INCREF(type); - } else { - assert(primitive == TYPED_OBJECT); - } - - if (!_PyObject_TypeCheckOptional(val, type, optional, exact)) { - PyErr_Format( - CiExc_StaticTypeError, - "%U expected '%s' for argument %U, got '%s'", - co->co_name, - type->tp_name, - idx < 0 ? PyTuple_GetItem(PyCode_GetCellvars(co), -(idx + 1)) - : PyTuple_GetItem(PyCode_GetVarnames(co), idx), - Py_TYPE(val)->tp_name); - Py_DECREF(type); - return -1; - } - - Py_DECREF(type); - - if (primitive <= TYPED_INT64) { - size_t value; - if (!_PyClassLoader_OverflowCheck(val, primitive, &value)) { - PyErr_SetString(PyExc_OverflowError, "int overflow"); - return -1; - } - } - } - return 0; - } - - _PyTypedArgsInfo* checks = - (_PyTypedArgsInfo*)co->co_mutable->shadow->arg_checks; - if (checks == NULL) { - // Shadow code is initialized, but we haven't cached the checks yet... - checks = _PyClassLoader_GetTypedArgsInfo(co, 0); - if (checks == NULL) { - return -1; - } - co->co_mutable->shadow->arg_checks = (PyObject*)checks; - } - - for (int i = 0; i < Py_SIZE(checks); i++) { - _PyTypedArgInfo* check = &checks->tai_args[i]; - long idx = check->tai_argnum; - PyObject* val; - // Look in freevars if necessary - if (idx < 0) { - assert(!_PyErr_Occurred(tstate)); - val = PyCell_GET(freevars[-(idx + 1)]); - } else { - val = fastlocals[idx]; - } - - if (!_PyObject_TypeCheckOptional( - val, check->tai_type, check->tai_optional, check->tai_exact)) { - PyErr_Format( - PyExc_TypeError, - "%U expected '%s' for argument %U, got '%s'", - co->co_name, - check->tai_type->tp_name, - idx < 0 ? PyTuple_GetItem(PyCode_GetCellvars(co), -(idx + 1)) - : PyTuple_GetItem(PyCode_GetVarnames(co), idx), - Py_TYPE(val)->tp_name); - return -1; - } - - if (check->tai_primitive_type != TYPED_OBJECT) { - size_t value; - if (!_PyClassLoader_OverflowCheck( - val, check->tai_primitive_type, &value)) { - PyErr_SetString(PyExc_OverflowError, "int overflow"); - - return -1; - } - } - } - return 0; -} - -static PyObject* _CiStaticEval_Vector( - PyThreadState* tstate, - PyFrameConstructor* con, - PyObject* locals, - PyObject* const* args, - size_t argcountf, - PyObject* kwnames, - int check_args) { - Py_ssize_t argcount = PyVectorcall_NARGS(argcountf); - Py_ssize_t awaited = Ci_Py_AWAITED_CALL(argcountf); - PyFrameObject* f = - Cix_PyEval_MakeFrameVector(tstate, con, locals, args, argcount, kwnames); - if (f == NULL) { - return NULL; - } - - PyCodeObject* co = (PyCodeObject*)con->fc_code; - assert(co->co_flags & CI_CO_STATICALLY_COMPILED); - if (check_args && _Ci_CheckArgs(tstate, f, co) < 0) { - Py_DECREF(f); - return NULL; - } - - const int co_flags = ((PyCodeObject*)con->fc_code)->co_flags; - if (awaited && (co_flags & CO_COROUTINE)) { - return _PyEval_EvalEagerCoro( - tstate, f, f->f_code->co_name, con->fc_qualname); - } - if (co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) { - return make_coro(con, f); - } - PyObject* retval = _PyEval_EvalFrame(tstate, f, 0); - - /* decref'ing the frame can cause __del__ methods to get invoked, - which can call back into Python. While we're done with the - current Python frame (f), the associated C stack is still in use, - so recursion_depth must be boosted for the duration. - */ - if (Py_REFCNT(f) > 1) { - Py_DECREF(f); - _PyObject_GC_TRACK(f); - } else { - ++tstate->recursion_depth; - Py_DECREF(f); - --tstate->recursion_depth; - } - return retval; -} - -PyObject* Ci_StaticFunction_Vectorcall( - PyObject* func, - PyObject* const* stack, - size_t nargsf, - PyObject* kwnames) { - assert(PyFunction_Check(func)); - PyFrameConstructor* f = PyFunction_AS_FRAME_CONSTRUCTOR(func); - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - Py_ssize_t awaited = Ci_Py_AWAITED_CALL(nargsf); - assert(nargs >= 0); - assert(nargs == 0 || stack != NULL); - - PyCodeObject* code = (PyCodeObject*)f->fc_code; - PyObject* globals = (code->co_flags & CO_OPTIMIZED) ? NULL : f->fc_globals; - - PyThreadState* tstate = _PyThreadState_GET(); - return _CiStaticEval_Vector( - tstate, f, globals, stack, nargs | awaited, kwnames, 1); -} - -PyObject* _Py_HOT_FUNCTION Ci_PyFunction_CallStatic( - PyFunctionObject* func, - PyObject* const* args, - size_t nargsf, - PyObject* kwnames) { - assert(PyFunction_Check(func)); - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 0 || args != NULL); - - PyThreadState* tstate = _PyThreadState_GET(); - assert(tstate != NULL); - - /* We are bound to a specific function that is known at compile time, and - * all of the arguments are guaranteed to be provided */ - PyCodeObject* co = (PyCodeObject*)func->func_code; - assert(co->co_argcount == nargs); - assert(co->co_flags & CI_CO_STATICALLY_COMPILED); - assert(co->co_flags & CO_OPTIMIZED); - assert(kwnames == NULL); - - /* Silence unused variable warnings. */ - (void)co; - (void)kwnames; - (void)nargs; - - PyFrameConstructor* con = PyFunction_AS_FRAME_CONSTRUCTOR(func); - - return _CiStaticEval_Vector(tstate, con, NULL, args, nargsf, NULL, 0); -} diff --git a/cinderx/Interpreter/3.12/Includes/ceval.c b/cinderx/Interpreter/3.12/Includes/ceval.c index 3aba725ae..06eee48fd 100644 --- a/cinderx/Interpreter/3.12/Includes/ceval.c +++ b/cinderx/Interpreter/3.12/Includes/ceval.c @@ -1575,8 +1575,11 @@ clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) tstate->c_recursion_remaining--; assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); _PyFrame_ClearExceptCode(frame); + _PyErr_ClearExcState(&gen->gi_exc_state); tstate->c_recursion_remaining++; frame->previous = NULL; + /* Inlined SEND bypasses gen_send_ex2(), so clear the awaiter here. */ + Py_CLEAR(gen->gi_ci_awaiter); } static void diff --git a/cinderx/Interpreter/3.12/Includes/ceval_macros.h b/cinderx/Interpreter/3.12/Includes/ceval_macros.h index e28f521cd..468c6db93 100644 --- a/cinderx/Interpreter/3.12/Includes/ceval_macros.h +++ b/cinderx/Interpreter/3.12/Includes/ceval_macros.h @@ -107,7 +107,6 @@ #define DISPATCH_INLINED(NEW_FRAME) \ do { \ - assert(tstate->interp->eval_frame == NULL); \ _PyFrame_SetStackPointer(frame, stack_pointer); \ frame->prev_instr = next_instr - 1; \ (NEW_FRAME)->previous = frame; \ diff --git a/cinderx/Interpreter/3.12/Includes/generated_cases.c.h b/cinderx/Interpreter/3.12/Includes/generated_cases.c.h index 02183cc49..d4035c740 100644 --- a/cinderx/Interpreter/3.12/Includes/generated_cases.c.h +++ b/cinderx/Interpreter/3.12/Includes/generated_cases.c.h @@ -3390,7 +3390,6 @@ PyObject *res2 = NULL; PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); - uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); #line 2656 "Python/bytecodes.c" /* Cached method object */ @@ -3400,9 +3399,6 @@ assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyHeapTypeObject *self_heap_type = (PyHeapTypeObject *)self_cls; - DEOPT_IF(self_heap_type->ht_cached_keys->dk_version != - keys_version, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); res2 = Py_NewRef(descr); diff --git a/cinderx/Interpreter/3.12/interpreter.c b/cinderx/Interpreter/3.12/interpreter.c index b15c95059..595495275 100644 --- a/cinderx/Interpreter/3.12/interpreter.c +++ b/cinderx/Interpreter/3.12/interpreter.c @@ -387,11 +387,8 @@ void Ci_InitOpcodes() { #define _PyOpcode_Caches _CiOpcode_Caches -bool Ci_DelayAdaptiveCode = false; -uint64_t Ci_AdaptiveThreshold = 80; - bool is_adaptive_enabled(CodeExtra *extra) { - return !Ci_DelayAdaptiveCode || extra->calls > Ci_AdaptiveThreshold; + return !Ci_GetDelayAdaptiveCode() || extra->calls > Ci_GetAdaptiveThreshold(); } static void diff --git a/cinderx/Interpreter/3.14/Includes/generated_cases.c.h b/cinderx/Interpreter/3.14/Includes/generated_cases.c.h index adcfa47c4..a0c487de2 100644 --- a/cinderx/Interpreter/3.14/Includes/generated_cases.c.h +++ b/cinderx/Interpreter/3.14/Includes/generated_cases.c.h @@ -393,14 +393,13 @@ assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyUnicode_Append(&temp, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - *target_local = PyStackRef_FromPyObjectSteal(temp); - _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(right_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*target_local)) { + if (temp == NULL) { + *target_local = PyStackRef_NULL; JUMP_TO_LABEL(error); } + *target_local = PyStackRef_FromPyObjectSteal(temp); #if TIER_ONE assert(next_instr->op.code == STORE_FAST); @@ -1574,7 +1573,9 @@ } if (Py_TYPE(callable_o) == &PyFunction_Type && !IS_PEP523_HOOKED(tstate) && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + FT_ATOMIC_LOAD_PTR_RELAXED( + ((PyFunctionObject *)callable_o)->vectorcall) == + _PyFunction_Vectorcall) { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); @@ -2629,7 +2630,9 @@ else { if (Py_TYPE(func) == &PyFunction_Type && !IS_PEP523_HOOKED(tstate) && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + FT_ATOMIC_LOAD_PTR_RELAXED( + ((PyFunctionObject *)func)->vectorcall) == + _PyFunction_Vectorcall) { PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); assert(PyTuple_CheckExact(callargs)); PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); @@ -2910,7 +2913,9 @@ int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); if (Py_TYPE(callable_o) == &PyFunction_Type && !IS_PEP523_HOOKED(tstate) && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + FT_ATOMIC_LOAD_PTR_RELAXED( + ((PyFunctionObject *)callable_o)->vectorcall) == + _PyFunction_Vectorcall) { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); @@ -6446,7 +6451,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); specialize_with_value(next_instr, func, INVOKE_FUNCTION_CACHED, 0, 0); stack_pointer = _PyFrame_GetStackPointer(frame); - } else { + } else if (_Py_IsImmortal(container)) { _PyFrame_SetStackPointer(frame, stack_pointer); PyObject** funcptr = _PyClassLoader_ResolveIndirectPtr(target); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7973,7 +7978,9 @@ } if (Py_TYPE(callable_o) == &PyFunction_Type && !IS_PEP523_HOOKED(tstate) && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + FT_ATOMIC_LOAD_PTR_RELAXED( + ((PyFunctionObject *)callable_o)->vectorcall) == + _PyFunction_Vectorcall) { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); @@ -8189,7 +8196,9 @@ else { if (Py_TYPE(func) == &PyFunction_Type && !IS_PEP523_HOOKED(tstate) && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + FT_ATOMIC_LOAD_PTR_RELAXED( + ((PyFunctionObject *)func)->vectorcall) == + _PyFunction_Vectorcall) { PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); assert(PyTuple_CheckExact(callargs)); PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); @@ -8342,7 +8351,9 @@ int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); if (Py_TYPE(callable_o) == &PyFunction_Type && !IS_PEP523_HOOKED(tstate) && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + FT_ATOMIC_LOAD_PTR_RELAXED( + ((PyFunctionObject *)callable_o)->vectorcall) == + _PyFunction_Vectorcall) { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); @@ -9449,11 +9460,26 @@ v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; #ifdef Py_GIL_DISABLED - - int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), - PyStackRef_AsPyObjectSteal(v)); + PyObject *lst = PyStackRef_AsPyObjectBorrow(list); + int err; + if (PyList_Check(lst)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyList_Append(lst, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = Ci_ListOrCheckedList_Append( + (PyListObject *)lst, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - JUMP_TO_LABEL(pop_1_error); + JUMP_TO_LABEL(error); } #else _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9468,10 +9494,7 @@ if (err < 0) { JUMP_TO_LABEL(error); } - stack_pointer += 1; #endif - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } @@ -9558,10 +9581,37 @@ // _LOAD_ATTR { self_or_null = &stack_pointer[0]; + #if PY_VERSION_HEX >= 0x030E0400 + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - PyObject *attr_o; if (oparg & 1) { - attr_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + attr = _Py_LoadAttr_StackRefSteal(tstate, owner, name, self_or_null); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(attr)) { + JUMP_TO_LABEL(pop_1_error); + } + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + JUMP_TO_LABEL(error); + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer += 1; + } + #else + + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + PyObject *attr_o = NULL; + if (oparg & 1) { _PyFrame_SetStackPointer(frame, stack_pointer); int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9597,6 +9647,7 @@ stack_pointer += 1; } attr = PyStackRef_FromPyObjectSteal(attr_o); + #endif } stack_pointer[-1] = attr; stack_pointer += (oparg&1); @@ -10008,18 +10059,7 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - } + /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_WITH_VALUES { PyObject *descr = read_obj(&this_instr[6].cache); @@ -10202,18 +10242,7 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - } + /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { PyObject *descr = read_obj(&this_instr[6].cache); @@ -10553,7 +10582,7 @@ INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); _PyStackRef value; assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + value = PyStackRef_FromPyObjectNew(Ci_common_consts[oparg]); stack_pointer[0] = value; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -11009,21 +11038,36 @@ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } - #ifdef META_PYTHON - if (PyLazyImport_CheckExact(res_o)) { + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + if (!increfed) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } + #ifdef META_PYTHON + if (PyLazyImport_CheckExact(PyStackRef_AsPyObjectBorrow(res))) { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(res); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (true) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + } #endif - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - if (!increfed) { + #else + #ifdef META_PYTHON + if (PyLazyImport_CheckExact(res_o)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } - #else + #endif res = PyStackRef_FromPyObjectNew(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); @@ -11083,21 +11127,36 @@ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } - #ifdef META_PYTHON - if (PyLazyImport_CheckExact(res_o)) { + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + if (!increfed) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } + #ifdef META_PYTHON + if (PyLazyImport_CheckExact(PyStackRef_AsPyObjectBorrow(res))) { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(res); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (true) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + } #endif - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - if (!increfed) { + #else + #ifdef META_PYTHON + if (PyLazyImport_CheckExact(res_o)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } - #else + #endif res = PyStackRef_FromPyObjectNew(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); @@ -12475,8 +12534,21 @@ _PyStackRef v; v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; + PyObject *set_o = PyStackRef_AsPyObjectBorrow(set); + if (!PySet_CheckExact(set_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'%T' object is not a set", set_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), + int err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(v)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { @@ -12509,6 +12581,22 @@ PyObject **ptr = (PyObject **)(((char *)func) + offset); assert(*ptr == NULL); *ptr = attr; + if (oparg == MAKE_FUNCTION_ANNOTATE && PyFunction_Check(attr)) { + PyFunctionObject *func_obj = (PyFunctionObject *)attr; + stack_pointer[-2] = func_out; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *fixed_qualname = PyUnicode_FromFormat("%U.__annotate__", ((PyFunctionObject *)func)->func_qualname); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (fixed_qualname == NULL) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_SETREF(func_obj->func_qualname, fixed_qualname); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 1; + } stack_pointer[-2] = func_out; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); diff --git a/cinderx/Interpreter/3.14/borrowed-ceval.c.template b/cinderx/Interpreter/3.14/borrowed-ceval.c.template index 76da57e2a..cb62a51bf 100644 --- a/cinderx/Interpreter/3.14/borrowed-ceval.c.template +++ b/cinderx/Interpreter/3.14/borrowed-ceval.c.template @@ -73,7 +73,6 @@ int _Py_CheckRecursiveCallPy(PyThreadState* tstate); // @Borrow function clear_thread_frame from Python/ceval.c // @Borrow function clear_gen_frame from Python/ceval.c // @Borrow function _PyEval_FrameClearAndPop from Python/ceval.c -// @Borrow function _PyEvalFramePushAndInit from Python/ceval.c // @Borrow function _PyEvalFramePushAndInit_Ex from Python/ceval.c // Cinder specific adapted functions diff --git a/cinderx/Interpreter/3.14/ceval.h b/cinderx/Interpreter/3.14/ceval.h index 8f36104b8..7a900d046 100644 --- a/cinderx/Interpreter/3.14/ceval.h +++ b/cinderx/Interpreter/3.14/ceval.h @@ -463,7 +463,7 @@ format_missing(PyThreadState *tstate, const char *kind, if (name_str == NULL) return; _PyErr_Format(tstate, PyExc_TypeError, - "%U() missing %i required %s argument%s: %U", + "%U() missing %zd required %s argument%s: %U", qualname, len, kind, @@ -979,41 +979,6 @@ _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) clear_gen_frame(tstate, frame); } } -_PyInterpreterFrame * -_PyEvalFramePushAndInit(PyThreadState *tstate, _PyStackRef func, - PyObject *locals, _PyStackRef const* args, - size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous) -{ - PyFunctionObject *func_obj = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(func); - PyCodeObject * code = (PyCodeObject *)func_obj->func_code; - CALL_STAT_INC(frames_pushed); - _PyInterpreterFrame *frame = _PyThreadState_PushFrame(tstate, code->co_framesize); - if (frame == NULL) { - goto fail; - } - _PyFrame_Initialize(tstate, frame, func, locals, code, 0, previous); - if (initialize_locals(tstate, func_obj, frame->localsplus, args, argcount, kwnames)) { - assert(frame->owner == FRAME_OWNED_BY_THREAD); - clear_thread_frame(tstate, frame); - return NULL; - } - return frame; -fail: - /* Consume the references */ - PyStackRef_CLOSE(func); - Py_XDECREF(locals); - for (size_t i = 0; i < argcount; i++) { - PyStackRef_CLOSE(args[i]); - } - if (kwnames) { - Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); - for (Py_ssize_t i = 0; i < kwcount; i++) { - PyStackRef_CLOSE(args[i+argcount]); - } - } - PyErr_NoMemory(); - return NULL; -} static _PyInterpreterFrame * _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func, PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous) diff --git a/cinderx/Interpreter/3.14/cinder-bytecodes.c b/cinderx/Interpreter/3.14/cinder-bytecodes.c index f17085f3b..885d1610d 100644 --- a/cinderx/Interpreter/3.14/cinder-bytecodes.c +++ b/cinderx/Interpreter/3.14/cinder-bytecodes.c @@ -45,6 +45,8 @@ #include "setobject.h" +#include "cinderx/module_c_state.h" + #define USE_COMPUTED_GOTOS 0 #include "ceval_macros.h" @@ -145,6 +147,13 @@ dummy_func( switch (opcode) { // BEGIN BYTECODES // + override inst(LOAD_COMMON_CONSTANT, ( -- value)) { + // Use our own copy of common constants to avoid depending on the + // offset of interp->common_consts within PyInterpreterState. + assert(oparg < NUM_COMMON_CONSTANTS); + value = PyStackRef_FromPyObjectNew(Ci_common_consts[oparg]); + } + override inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); @@ -187,7 +196,9 @@ dummy_func( // Check if the call can be inlined or not if (Py_TYPE(callable_o) == &PyFunction_Type && !IS_PEP523_HOOKED(tstate) && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + FT_ATOMIC_LOAD_PTR_RELAXED( + ((PyFunctionObject *)callable_o)->vectorcall) == + _PyFunction_Vectorcall) { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); @@ -281,7 +292,9 @@ dummy_func( // Check if the call can be inlined or not if (Py_TYPE(callable_o) == &PyFunction_Type && !IS_PEP523_HOOKED(tstate) && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + FT_ATOMIC_LOAD_PTR_RELAXED( + ((PyFunctionObject *)callable_o)->vectorcall) == + _PyFunction_Vectorcall) { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); @@ -380,7 +393,9 @@ dummy_func( else { if (Py_TYPE(func) == &PyFunction_Type && !IS_PEP523_HOOKED(tstate) && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + FT_ATOMIC_LOAD_PTR_RELAXED( + ((PyFunctionObject *)func)->vectorcall) == + _PyFunction_Vectorcall) { PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); assert(PyTuple_CheckExact(callargs)); PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); @@ -492,9 +507,17 @@ dummy_func( override inst(LIST_APPEND, (list, unused[oparg-1], v -- list, unused[oparg-1])) { #ifdef Py_GIL_DISABLED - // T250369690: Need thread-safe checked collections - int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), - PyStackRef_AsPyObjectSteal(v)); + PyObject *lst = PyStackRef_AsPyObjectBorrow(list); + int err; + if (PyList_Check(lst)) { + err = PyList_Append(lst, PyStackRef_AsPyObjectBorrow(v)); + } + else { + // T250369690: Need thread-safe checked collections + err = Ci_ListOrCheckedList_Append( + (PyListObject *)lst, PyStackRef_AsPyObjectBorrow(v)); + } + PyStackRef_CLOSE(v); ERROR_IF(err < 0); #else int err = Ci_ListOrCheckedList_Append( @@ -504,6 +527,62 @@ dummy_func( #endif } + override op(_LOAD_ATTR, (owner -- attr, self_or_null[oparg&1])) { +#if PY_VERSION_HEX >= 0x030E0400 + // New version in Python 3.14.4 that uses _Py_LoadAttr_StackRefSteal. + + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + if (oparg & 1) { + /* Designed to work in tandem with CALL, pushes two values. */ + attr = _Py_LoadAttr_StackRefSteal(tstate, owner, name, self_or_null); + DEAD(owner); + ERROR_IF(PyStackRef_IsNull(attr)); + } + else { + /* Classic, pushes one value. */ + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + PyStackRef_CLOSE(owner); + ERROR_IF(attr_o == NULL); + attr = PyStackRef_FromPyObjectSteal(attr_o); + } +#else + // Older version pre-3.14.4. + + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + PyObject *attr_o = NULL; + if (oparg & 1) { + int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + if (is_meth) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr_o != NULL); // No errors on this branch + self_or_null[0] = owner; // Transfer ownership + DEAD(owner); + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + meth | NULL | arg1 | ... | argN + */ + PyStackRef_CLOSE(owner); + ERROR_IF(attr_o == NULL); + self_or_null[0] = PyStackRef_NULL; + } + } + else { + /* Classic, pushes one value. */ + attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + PyStackRef_CLOSE(owner); + ERROR_IF(attr_o == NULL); + } + attr = PyStackRef_FromPyObjectSteal(attr_o); +#endif + } + override inst(EXTENDED_OPCODE, (args[oparg>>2] -- top[oparg&0x03])) { // Decode any extended oparg int extop = (int)next_instr->op.code; diff --git a/cinderx/Interpreter/3.14/cinder_opcode_ids.h b/cinderx/Interpreter/3.14/cinder_opcode_ids.h index 4ecd9f9dc..9a8c77310 100644 --- a/cinderx/Interpreter/3.14/cinder_opcode_ids.h +++ b/cinderx/Interpreter/3.14/cinder_opcode_ids.h @@ -1,7 +1,7 @@ // Copyright (c) Meta Platforms, Inc. and affiliates. -// 3.14 has a simple file that just defines the relavant ids: +// 3.14 has a simple file that just defines the relevant ids: #include "opcode.h" diff --git a/cinderx/Interpreter/3.14/interpreter.c b/cinderx/Interpreter/3.14/interpreter.c index a49a0e376..53f9469d2 100644 --- a/cinderx/Interpreter/3.14/interpreter.c +++ b/cinderx/Interpreter/3.14/interpreter.c @@ -32,13 +32,42 @@ #define DK_KIND(dk) (dk->dk_kind) #endif -#ifdef ENABLE_INTERPRETER_LOOP +#undef EXTRA_CASES + +#define EXTRA_CASES \ + case 122: \ + case 123: \ + case 124: \ + case 125: \ + case 127: \ + case 212: \ + case 213: \ + case 214: \ + case 215: \ + case 216: \ + case 217: \ + case 218: \ + case 219: \ + case 220: \ + case 221: \ + case 222: \ + case 223: \ + case 224: \ + case 225: \ + case 226: \ + case 227: \ + case 228: \ + case 229: \ + case 230: \ + case 231: \ + case 232: \ + case 233: \ + ; -bool Ci_DelayAdaptiveCode = false; -uint64_t Ci_AdaptiveThreshold = 80; +#ifdef ENABLE_INTERPRETER_LOOP bool is_adaptive_enabled(CodeExtra *extra) { - return !Ci_DelayAdaptiveCode || Ci_code_extra_get_calls(extra) > Ci_AdaptiveThreshold; + return !Ci_GetDelayAdaptiveCode() || Ci_code_extra_get_calls(extra) > Ci_GetAdaptiveThreshold(); } #endif diff --git a/cinderx/Interpreter/3.14/opcode.h b/cinderx/Interpreter/3.14/opcode.h deleted file mode 100644 index e0eceb46a..000000000 --- a/cinderx/Interpreter/3.14/opcode.h +++ /dev/null @@ -1,261 +0,0 @@ -// This file is generated by Tools/cases_generator/opcode_id_generator.py -// from: -// Python/bytecodes.c, 3.14/cinder-bytecodes.c -// Do not edit! - -#ifndef Py_OPCODE_IDS_H -#define Py_OPCODE_IDS_H -#ifdef __cplusplus -extern "C" { -#endif - -/* Instruction opcodes for compiled code */ -#define CACHE 0 -#define BINARY_SLICE 1 -#define BUILD_TEMPLATE 2 -#define BINARY_OP_INPLACE_ADD_UNICODE 3 -#define CALL_FUNCTION_EX 4 -#define CHECK_EG_MATCH 5 -#define CHECK_EXC_MATCH 6 -#define CLEANUP_THROW 7 -#define DELETE_SUBSCR 8 -#define END_FOR 9 -#define END_SEND 10 -#define EXIT_INIT_CHECK 11 -#define FORMAT_SIMPLE 12 -#define FORMAT_WITH_SPEC 13 -#define GET_AITER 14 -#define GET_ANEXT 15 -#define GET_ITER 16 -#define RESERVED 17 -#define GET_LEN 18 -#define GET_YIELD_FROM_ITER 19 -#define INTERPRETER_EXIT 20 -#define LOAD_BUILD_CLASS 21 -#define LOAD_LOCALS 22 -#define MAKE_FUNCTION 23 -#define MATCH_KEYS 24 -#define MATCH_MAPPING 25 -#define MATCH_SEQUENCE 26 -#define NOP 27 -#define NOT_TAKEN 28 -#define POP_EXCEPT 29 -#define POP_ITER 30 -#define POP_TOP 31 -#define PUSH_EXC_INFO 32 -#define PUSH_NULL 33 -#define RETURN_GENERATOR 34 -#define RETURN_VALUE 35 -#define SETUP_ANNOTATIONS 36 -#define STORE_SLICE 37 -#define STORE_SUBSCR 38 -#define TO_BOOL 39 -#define UNARY_INVERT 40 -#define UNARY_NEGATIVE 41 -#define UNARY_NOT 42 -#define WITH_EXCEPT_START 43 -#define BINARY_OP 44 -#define BUILD_INTERPOLATION 45 -#define BUILD_LIST 46 -#define BUILD_MAP 47 -#define BUILD_SET 48 -#define BUILD_SLICE 49 -#define BUILD_STRING 50 -#define BUILD_TUPLE 51 -#define CALL 52 -#define CALL_INTRINSIC_1 53 -#define CALL_INTRINSIC_2 54 -#define CALL_KW 55 -#define COMPARE_OP 56 -#define CONTAINS_OP 57 -#define CONVERT_VALUE 58 -#define COPY 59 -#define COPY_FREE_VARS 60 -#define DELETE_ATTR 61 -#define DELETE_DEREF 62 -#define DELETE_FAST 63 -#define DELETE_GLOBAL 64 -#define DELETE_NAME 65 -#define DICT_MERGE 66 -#define DICT_UPDATE 67 -#define END_ASYNC_FOR 68 -#define EXTENDED_ARG 69 -#define FOR_ITER 70 -#define GET_AWAITABLE 71 -#define IMPORT_FROM 72 -#define IMPORT_NAME 73 -#define IS_OP 74 -#define JUMP_BACKWARD 75 -#define JUMP_BACKWARD_NO_INTERRUPT 76 -#define JUMP_FORWARD 77 -#define LIST_APPEND 78 -#define LIST_EXTEND 79 -#define LOAD_ATTR 80 -#define LOAD_COMMON_CONSTANT 81 -#define LOAD_CONST 82 -#define LOAD_DEREF 83 -#define LOAD_FAST 84 -#define LOAD_FAST_AND_CLEAR 85 -#define LOAD_FAST_BORROW 86 -#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 87 -#define LOAD_FAST_CHECK 88 -#define LOAD_FAST_LOAD_FAST 89 -#define LOAD_FROM_DICT_OR_DEREF 90 -#define LOAD_FROM_DICT_OR_GLOBALS 91 -#define LOAD_GLOBAL 92 -#define LOAD_NAME 93 -#define LOAD_SMALL_INT 94 -#define LOAD_SPECIAL 95 -#define LOAD_SUPER_ATTR 96 -#define MAKE_CELL 97 -#define MAP_ADD 98 -#define MATCH_CLASS 99 -#define POP_JUMP_IF_FALSE 100 -#define POP_JUMP_IF_NONE 101 -#define POP_JUMP_IF_NOT_NONE 102 -#define POP_JUMP_IF_TRUE 103 -#define RAISE_VARARGS 104 -#define RERAISE 105 -#define SEND 106 -#define SET_ADD 107 -#define SET_FUNCTION_ATTRIBUTE 108 -#define SET_UPDATE 109 -#define STORE_ATTR 110 -#define STORE_DEREF 111 -#define STORE_FAST 112 -#define STORE_FAST_LOAD_FAST 113 -#define STORE_FAST_STORE_FAST 114 -#define STORE_GLOBAL 115 -#define STORE_NAME 116 -#define SWAP 117 -#define UNPACK_EX 118 -#define UNPACK_SEQUENCE 119 -#define YIELD_VALUE 120 -#define EAGER_IMPORT_NAME 121 -#define EXTENDED_OPCODE 126 -#define RESUME 128 -#define BINARY_OP_ADD_FLOAT 129 -#define BINARY_OP_ADD_INT 130 -#define BINARY_OP_ADD_UNICODE 131 -#define BINARY_OP_EXTEND 132 -#define BINARY_OP_MULTIPLY_FLOAT 133 -#define BINARY_OP_MULTIPLY_INT 134 -#define BINARY_OP_SUBSCR_DICT 135 -#define BINARY_OP_SUBSCR_GETITEM 136 -#define BINARY_OP_SUBSCR_LIST_INT 137 -#define BINARY_OP_SUBSCR_LIST_SLICE 138 -#define BINARY_OP_SUBSCR_STR_INT 139 -#define BINARY_OP_SUBSCR_TUPLE_INT 140 -#define BINARY_OP_SUBTRACT_FLOAT 141 -#define BINARY_OP_SUBTRACT_INT 142 -#define CALL_ALLOC_AND_ENTER_INIT 143 -#define CALL_BOUND_METHOD_EXACT_ARGS 144 -#define CALL_BOUND_METHOD_GENERAL 145 -#define CALL_BUILTIN_CLASS 146 -#define CALL_BUILTIN_FAST 147 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 148 -#define CALL_BUILTIN_O 149 -#define CALL_ISINSTANCE 150 -#define CALL_KW_BOUND_METHOD 151 -#define CALL_KW_NON_PY 152 -#define CALL_KW_PY 153 -#define CALL_LEN 154 -#define CALL_LIST_APPEND 155 -#define CALL_METHOD_DESCRIPTOR_FAST 156 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 157 -#define CALL_METHOD_DESCRIPTOR_NOARGS 158 -#define CALL_METHOD_DESCRIPTOR_O 159 -#define CALL_NON_PY_GENERAL 160 -#define CALL_PY_EXACT_ARGS 161 -#define CALL_PY_GENERAL 162 -#define CALL_STR_1 163 -#define CALL_TUPLE_1 164 -#define CALL_TYPE_1 165 -#define COMPARE_OP_FLOAT 166 -#define COMPARE_OP_INT 167 -#define COMPARE_OP_STR 168 -#define CONTAINS_OP_DICT 169 -#define CONTAINS_OP_SET 170 -#define FOR_ITER_GEN 171 -#define FOR_ITER_LIST 172 -#define FOR_ITER_RANGE 173 -#define FOR_ITER_TUPLE 174 -#define JUMP_BACKWARD_JIT 175 -#define JUMP_BACKWARD_NO_JIT 176 -#define LOAD_ATTR_CLASS 177 -#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 178 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 179 -#define LOAD_ATTR_INSTANCE_VALUE 180 -#define LOAD_ATTR_METHOD_LAZY_DICT 181 -#define LOAD_ATTR_METHOD_NO_DICT 182 -#define LOAD_ATTR_METHOD_WITH_VALUES 183 -#define LOAD_ATTR_MODULE 184 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 185 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 186 -#define LOAD_ATTR_PROPERTY 187 -#define LOAD_ATTR_SLOT 188 -#define LOAD_ATTR_WITH_HINT 189 -#define LOAD_CONST_IMMORTAL 190 -#define LOAD_CONST_MORTAL 191 -#define LOAD_GLOBAL_BUILTIN 192 -#define LOAD_GLOBAL_MODULE 193 -#define LOAD_SUPER_ATTR_ATTR 194 -#define LOAD_SUPER_ATTR_METHOD 195 -#define RESUME_CHECK 196 -#define SEND_GEN 197 -#define STORE_ATTR_INSTANCE_VALUE 198 -#define STORE_ATTR_SLOT 199 -#define STORE_ATTR_WITH_HINT 200 -#define STORE_SUBSCR_DICT 201 -#define STORE_SUBSCR_LIST_INT 202 -#define TO_BOOL_ALWAYS_TRUE 203 -#define TO_BOOL_BOOL 204 -#define TO_BOOL_INT 205 -#define TO_BOOL_LIST 206 -#define TO_BOOL_NONE 207 -#define TO_BOOL_STR 208 -#define UNPACK_SEQUENCE_LIST 209 -#define UNPACK_SEQUENCE_TUPLE 210 -#define UNPACK_SEQUENCE_TWO_TUPLE 211 -#define INSTRUMENTED_END_FOR 234 -#define INSTRUMENTED_POP_ITER 235 -#define INSTRUMENTED_END_SEND 236 -#define INSTRUMENTED_FOR_ITER 237 -#define INSTRUMENTED_INSTRUCTION 238 -#define INSTRUMENTED_JUMP_FORWARD 239 -#define INSTRUMENTED_NOT_TAKEN 240 -#define INSTRUMENTED_POP_JUMP_IF_TRUE 241 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 242 -#define INSTRUMENTED_POP_JUMP_IF_NONE 243 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 244 -#define INSTRUMENTED_RESUME 245 -#define INSTRUMENTED_RETURN_VALUE 246 -#define INSTRUMENTED_YIELD_VALUE 247 -#define INSTRUMENTED_END_ASYNC_FOR 248 -#define INSTRUMENTED_LOAD_SUPER_ATTR 249 -#define INSTRUMENTED_CALL 250 -#define INSTRUMENTED_CALL_KW 251 -#define INSTRUMENTED_CALL_FUNCTION_EX 252 -#define INSTRUMENTED_JUMP_BACKWARD 253 -#define INSTRUMENTED_LINE 254 -#define ENTER_EXECUTOR 255 -#define ANNOTATIONS_PLACEHOLDER 256 -#define JUMP 257 -#define JUMP_IF_FALSE 258 -#define JUMP_IF_TRUE 259 -#define JUMP_NO_INTERRUPT 260 -#define LOAD_CLOSURE 261 -#define POP_BLOCK 262 -#define SETUP_CLEANUP 263 -#define SETUP_FINALLY 264 -#define SETUP_WITH 265 -#define STORE_FAST_MAYBE_NULL 266 - -#define HAVE_ARGUMENT 43 -#define MIN_SPECIALIZED_OPCODE 129 -#define MIN_INSTRUMENTED_OPCODE 234 - -#ifdef __cplusplus -} -#endif -#endif /* !Py_OPCODE_IDS_H */ diff --git a/cinderx/Interpreter/3.15/Includes/Python/ceval.h b/cinderx/Interpreter/3.15/Includes/Python/ceval.h index bb5f7ddb8..0437ab85c 100644 --- a/cinderx/Interpreter/3.15/Includes/Python/ceval.h +++ b/cinderx/Interpreter/3.15/Includes/Python/ceval.h @@ -367,7 +367,7 @@ no_tools_for_global_event(PyThreadState *tstate, int event) static inline bool no_tools_for_local_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event) { - assert(event < _PY_MONITORING_LOCAL_EVENTS); + assert(event < _PY_MONITORING_UNGROUPED_EVENTS); _PyCoMonitoringData *data = _PyFrame_GetCode(frame)->_co_monitoring; if (data) { return data->active_monitors.tools[event] == 0; @@ -382,7 +382,7 @@ monitor_handled(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *exc) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { return 0; } return _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); @@ -393,7 +393,7 @@ monitor_throw(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_THROW)) { + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_PY_THROW)) { return; } do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW); @@ -403,7 +403,7 @@ static void monitor_reraise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RERAISE)) { + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_RERAISE)) { return; } do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RERAISE); @@ -431,7 +431,7 @@ monitor_unwind(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) { - if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_UNWIND)) { + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_PY_UNWIND)) { return; } do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_UNWIND); diff --git a/cinderx/Interpreter/3.15/Includes/Python/ceval_macros.h b/cinderx/Interpreter/3.15/Includes/Python/ceval_macros.h index b127812b4..1ebbc096d 100644 --- a/cinderx/Interpreter/3.15/Includes/Python/ceval_macros.h +++ b/cinderx/Interpreter/3.15/Includes/Python/ceval_macros.h @@ -168,7 +168,6 @@ #define STOP_TRACING() ((void)(0)); #endif - /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ #ifdef Py_DEBUG #define PRE_DISPATCH_GOTO() if (frame->lltrace >= 5) { \ @@ -220,14 +219,14 @@ do { \ DISPATCH_GOTO_NON_TRACING(); \ } -#define DISPATCH_INLINED(NEW_FRAME) \ - do { \ - assert(tstate->interp->eval_frame == NULL); \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - assert((NEW_FRAME)->previous == frame); \ - frame = tstate->current_frame = (NEW_FRAME); \ - CALL_STAT_INC(inlined_py_calls); \ - JUMP_TO_LABEL(start_frame); \ +#define DISPATCH_INLINED(NEW_FRAME) \ + do { \ + assert(!IS_PEP523_HOOKED(tstate)); \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + assert((NEW_FRAME)->previous == frame); \ + frame = tstate->current_frame = (NEW_FRAME); \ + CALL_STAT_INC(inlined_py_calls); \ + JUMP_TO_LABEL(start_frame); \ } while (0) /* Tuple access macros */ @@ -329,11 +328,24 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define CONSTS() _PyFrame_GetCode(frame)->co_consts #define NAMES() _PyFrame_GetCode(frame)->co_names +#if defined(WITH_DTRACE) && !defined(Py_BUILD_CORE_MODULE) +static void dtrace_function_entry(_PyInterpreterFrame *); +static void dtrace_function_return(_PyInterpreterFrame *); + #define DTRACE_FUNCTION_ENTRY() \ if (PyDTrace_FUNCTION_ENTRY_ENABLED()) { \ dtrace_function_entry(frame); \ } +#define DTRACE_FUNCTION_RETURN() \ + if (PyDTrace_FUNCTION_RETURN_ENABLED()) { \ + dtrace_function_return(frame); \ + } +#else +#define DTRACE_FUNCTION_ENTRY() ((void)0) +#define DTRACE_FUNCTION_RETURN() ((void)0) +#endif + /* This takes a uint16_t instead of a _Py_BackoffCounter, * because it is used directly on the cache entry in generated code, * which is always an integral type. */ @@ -376,14 +388,15 @@ GETITEM(PyObject *v, Py_ssize_t i) { // for an exception handler, displaying the traceback, and so on #define INSTRUMENTED_JUMP(src, dest, event) \ do { \ + _Py_CODEUNIT *_dest = (dest); \ if (tstate->tracing) {\ - next_instr = dest; \ + next_instr = _dest; \ } else { \ _PyFrame_SetStackPointer(frame, stack_pointer); \ - next_instr = _Py_call_instrumentation_jump(this_instr, tstate, event, frame, src, dest); \ + next_instr = _Py_call_instrumentation_jump(this_instr, tstate, event, frame, src, _dest); \ stack_pointer = _PyFrame_GetStackPointer(frame); \ if (next_instr == NULL) { \ - next_instr = (dest)+1; \ + next_instr = _dest + 1; \ JUMP_TO_LABEL(error); \ } \ } \ @@ -514,6 +527,22 @@ check_periodics(PyThreadState *tstate) { return 0; } +static inline int +check_periodics_at_end(PyThreadState *tstate, _PyInterpreterFrame *frame) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + // Do not handle pending interrupts if the previous instruction was LOAD_SPECIAL + // This may also not handle interrupts if a cache looks like LOAD_SPECIAL, + // but this is benign as we won't skip periodic checks indefinitely. + if (frame->instr_ptr[-1].op.code == LOAD_SPECIAL) { + return 0; + } + return _Py_HandlePending(tstate); + } + return 0; +} + // Mark the generator as executing. Returns true if the state was changed, // false if it was already executing or finished. static inline bool @@ -543,3 +572,91 @@ gen_try_set_executing(PyGenObject *gen) } return false; } + +// Macro for inplace float binary ops (tier 2 only). +// Mutates the uniquely-referenced TARGET operand in place. +// TARGET must be either left or right. +#define FLOAT_INPLACE_OP(left, right, TARGET, OP) \ + do { \ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); \ + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); \ + assert(PyFloat_CheckExact(left_o)); \ + assert(PyFloat_CheckExact(right_o)); \ + assert(_PyObject_IsUniquelyReferenced( \ + PyStackRef_AsPyObjectBorrow(TARGET))); \ + STAT_INC(BINARY_OP, hit); \ + double _dres = \ + ((PyFloatObject *)left_o)->ob_fval \ + OP ((PyFloatObject *)right_o)->ob_fval; \ + ((PyFloatObject *)PyStackRef_AsPyObjectBorrow(TARGET)) \ + ->ob_fval = _dres; \ + } while (0) + +// Inplace float true division. Sets _divop_err to 1 on zero division. +// Caller must check _divop_err and call ERROR_NO_POP() if set. +#define FLOAT_INPLACE_DIVOP(left, right, TARGET) \ + int _divop_err = 0; \ + do { \ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); \ + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); \ + assert(PyFloat_CheckExact(left_o)); \ + assert(PyFloat_CheckExact(right_o)); \ + assert(_PyObject_IsUniquelyReferenced( \ + PyStackRef_AsPyObjectBorrow(TARGET))); \ + STAT_INC(BINARY_OP, hit); \ + double _divisor = ((PyFloatObject *)right_o)->ob_fval; \ + if (_divisor == 0.0) { \ + PyErr_SetString(PyExc_ZeroDivisionError, \ + "float division by zero"); \ + _divop_err = 1; \ + break; \ + } \ + double _dres = ((PyFloatObject *)left_o)->ob_fval / _divisor; \ + ((PyFloatObject *)PyStackRef_AsPyObjectBorrow(TARGET)) \ + ->ob_fval = _dres; \ + } while (0) + +// Inplace compact int operation. TARGET is expected to be uniquely +// referenced at the optimizer level, but at runtime it may be a +// cached small int singleton. We check _Py_IsImmortal on TARGET +// to decide whether inplace mutation is safe. +// +// After the macro, _int_inplace_res holds the result (may be NULL +// on allocation failure). On success, TARGET was mutated in place +// and _int_inplace_res is a DUP'd reference to it. On fallback +// (small int target, small int result, or overflow), _int_inplace_res +// is from FUNC (_PyCompactLong_Add etc.). +// FUNC is the fallback function (_PyCompactLong_Add etc.) +#define INT_INPLACE_OP(left, right, TARGET, OP, FUNC) \ + _PyStackRef _int_inplace_res = PyStackRef_NULL; \ + do { \ + PyObject *target_o = PyStackRef_AsPyObjectBorrow(TARGET); \ + if (_Py_IsImmortal(target_o)) { \ + break; \ + } \ + assert(_PyObject_IsUniquelyReferenced(target_o)); \ + Py_ssize_t left_val = _PyLong_CompactValue( \ + (PyLongObject *)PyStackRef_AsPyObjectBorrow(left)); \ + Py_ssize_t right_val = _PyLong_CompactValue( \ + (PyLongObject *)PyStackRef_AsPyObjectBorrow(right)); \ + Py_ssize_t result = left_val OP right_val; \ + if (!_PY_IS_SMALL_INT(result) \ + && ((twodigits)((stwodigits)result) + PyLong_MASK \ + < (twodigits)PyLong_MASK + PyLong_BASE)) \ + { \ + _PyLong_SetSignAndDigitCount( \ + (PyLongObject *)target_o, result < 0 ? -1 : 1, 1); \ + ((PyLongObject *)target_o)->long_value.ob_digit[0] = \ + (digit)(result < 0 ? -result : result); \ + _int_inplace_res = PyStackRef_DUP(TARGET); \ + break; \ + } \ + } while (0); \ + if (PyStackRef_IsNull(_int_inplace_res)) { \ + _int_inplace_res = FUNC( \ + (PyLongObject *)PyStackRef_AsPyObjectBorrow(left), \ + (PyLongObject *)PyStackRef_AsPyObjectBorrow(right)); \ + } + +#define CALL_TP_ITERITEM_NO_ESCAPE(ITER, INDEX) \ + Py_TYPE(ITER)->_tp_iteritem((ITER), (INDEX)) diff --git a/cinderx/Interpreter/3.15/Includes/ceval_macros.h b/cinderx/Interpreter/3.15/Includes/ceval_macros.h new file mode 100644 index 000000000..8a2ba40c1 --- /dev/null +++ b/cinderx/Interpreter/3.15/Includes/ceval_macros.h @@ -0,0 +1,662 @@ +// Macros and other things needed by ceval.c, and bytecodes.c + +/* Computed GOTOs, or + the-optimization-commonly-but-improperly-known-as-"threaded code" + using gcc's labels-as-values extension + (http://gcc.gnu.org/onlinedocs/gcc/Labels-as-Values.html). + + The traditional bytecode evaluation loop uses a "switch" statement, which + decent compilers will optimize as a single indirect branch instruction + combined with a lookup table of jump addresses. However, since the + indirect jump instruction is shared by all opcodes, the CPU will have a + hard time making the right prediction for where to jump next (actually, + it will be always wrong except in the uncommon case of a sequence of + several identical opcodes). + + "Threaded code" in contrast, uses an explicit jump table and an explicit + indirect jump instruction at the end of each opcode. Since the jump + instruction is at a different address for each opcode, the CPU will make a + separate prediction for each of these instructions, which is equivalent to + predicting the second opcode of each opcode pair. These predictions have + a much better chance to turn out valid, especially in small bytecode loops. + + A mispredicted branch on a modern CPU flushes the whole pipeline and + can cost several CPU cycles (depending on the pipeline depth), + and potentially many more instructions (depending on the pipeline width). + A correctly predicted branch, however, is nearly free. + + At the time of this writing, the "threaded code" version is up to 15-20% + faster than the normal "switch" version, depending on the compiler and the + CPU architecture. + + NOTE: care must be taken that the compiler doesn't try to "optimize" the + indirect jumps by sharing them between all opcodes. Such optimizations + can be disabled on gcc by using the -fno-gcse flag (or possibly + -fno-crossjumping). +*/ + +/* Use macros rather than inline functions, to make it as clear as possible + * to the C compiler that the tracing check is a simple test then branch. + * We want to be sure that the compiler knows this before it generates + * the CFG. + */ + +#ifdef WITH_DTRACE +#define OR_DTRACE_LINE | (PyDTrace_LINE_ENABLED() ? 255 : 0) +#else +#define OR_DTRACE_LINE +#endif + +#ifdef HAVE_COMPUTED_GOTOS + #ifndef USE_COMPUTED_GOTOS + #define USE_COMPUTED_GOTOS 1 + #endif +#else + #if defined(USE_COMPUTED_GOTOS) && USE_COMPUTED_GOTOS + #error "Computed gotos are not supported on this compiler." + #endif + #undef USE_COMPUTED_GOTOS + #define USE_COMPUTED_GOTOS 0 +#endif + +#ifdef Py_STATS +#define INSTRUCTION_STATS(op) \ + do { \ + PyStats *s = _PyStats_GET(); \ + OPCODE_EXE_INC(op); \ + if (s) s->opcode_stats[lastopcode].pair_count[op]++; \ + lastopcode = op; \ + } while (0) +#else +#define INSTRUCTION_STATS(op) ((void)0) +#endif + +#ifdef Py_STATS +# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, const void *instruction_funcptr_table, int oparg, int lastopcode, bool adaptive_enabled +# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg, lastopcode, adaptive_enabled +#else +# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, const void *instruction_funcptr_table, int oparg, bool adaptive_enabled +# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg, adaptive_enabled +#endif + +#if _Py_TAIL_CALL_INTERP +# if defined(__clang__) || defined(__GNUC__) +# if !_Py__has_attribute(preserve_none) || !_Py__has_attribute(musttail) +# error "This compiler does not have support for efficient tail calling." +# endif +# elif defined(_MSC_VER) && (_MSC_VER < 1950) +# error "You need at least VS 2026 / PlatformToolset v145 for tail calling." +# endif +# if defined(_MSC_VER) && !defined(__clang__) +# define Py_MUSTTAIL [[msvc::musttail]] +# define Py_PRESERVE_NONE_CC __preserve_none +# else +# define Py_MUSTTAIL __attribute__((musttail)) +# define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) +# endif + typedef PyObject *(Py_PRESERVE_NONE_CC *py_tail_call_funcptr)(TAIL_CALL_PARAMS); + +# define DISPATCH_TABLE_VAR instruction_funcptr_table +# define DISPATCH_TABLE instruction_funcptr_handler_table +# define TRACING_DISPATCH_TABLE instruction_funcptr_tracing_table +# define TARGET(op) Py_NO_INLINE PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_##op(TAIL_CALL_PARAMS) + +# define DISPATCH_GOTO() \ + do { \ + Py_MUSTTAIL return (((py_tail_call_funcptr *)instruction_funcptr_table)[opcode])(TAIL_CALL_ARGS); \ + } while (0) +# define DISPATCH_GOTO_NON_TRACING() \ + do { \ + Py_MUSTTAIL return (((py_tail_call_funcptr *)DISPATCH_TABLE)[opcode])(TAIL_CALL_ARGS); \ + } while (0) +# define JUMP_TO_LABEL(name) \ + do { \ + Py_MUSTTAIL return (_TAIL_CALL_##name)(TAIL_CALL_ARGS); \ + } while (0) +# ifdef Py_STATS +# define JUMP_TO_PREDICTED(name) \ + do { \ + Py_MUSTTAIL return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, this_instr, instruction_funcptr_table, oparg, lastopcode, adaptive_enabled); \ + } while (0) +# else +# define JUMP_TO_PREDICTED(name) \ + do { \ + Py_MUSTTAIL return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, this_instr, instruction_funcptr_table, oparg, adaptive_enabled); \ + } while (0) +# endif +# define LABEL(name) TARGET(name) +#elif USE_COMPUTED_GOTOS +# define DISPATCH_TABLE_VAR opcode_targets +# define DISPATCH_TABLE opcode_targets_table +# define TRACING_DISPATCH_TABLE opcode_tracing_targets_table +# define TARGET(op) TARGET_##op: +# define DISPATCH_GOTO() goto *opcode_targets[opcode] +# define DISPATCH_GOTO_NON_TRACING() goto *DISPATCH_TABLE[opcode]; +# define JUMP_TO_LABEL(name) goto name; +# define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; +# define LABEL(name) name: +#else +# define TARGET(op) case op: TARGET_##op: +# define DISPATCH_GOTO() dispatch_code = opcode | tracing_mode ; goto dispatch_opcode +# define DISPATCH_GOTO_NON_TRACING() dispatch_code = opcode; goto dispatch_opcode +# define JUMP_TO_LABEL(name) goto name; +# define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; +# define LABEL(name) name: +#endif + +#if (_Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS) && _Py_TIER2 +# define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) +# define ENTER_TRACING() \ + DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; +# define LEAVE_TRACING() \ + DISPATCH_TABLE_VAR = DISPATCH_TABLE; +#else +# define IS_JIT_TRACING() (tracing_mode != 0) +# define ENTER_TRACING() tracing_mode = 255 +# define LEAVE_TRACING() tracing_mode = 0 +#endif + +#if _Py_TIER2 +#define STOP_TRACING() \ + do { \ + if (IS_JIT_TRACING()) { \ + LEAVE_TRACING(); \ + _PyJit_FinalizeTracing(tstate, 0); \ + } \ + } while (0); +#else +#define STOP_TRACING() ((void)(0)); +#endif + +/* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ +#ifdef Py_DEBUG +#define PRE_DISPATCH_GOTO() if (frame->lltrace >= 5) { \ + lltrace_instruction(frame, stack_pointer, next_instr, opcode, oparg); } +#else +#define PRE_DISPATCH_GOTO() ((void)0) +#endif + +#ifdef Py_DEBUG +#define LLTRACE_RESUME_FRAME() \ +do { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + frame->lltrace = lltrace; \ +} while (0) +#else +#define LLTRACE_RESUME_FRAME() ((void)0) +#endif + +#ifdef Py_GIL_DISABLED +#define QSBR_QUIESCENT_STATE(tstate) _Py_qsbr_quiescent_state(((_PyThreadStateImpl *)tstate)->qsbr) +#else +#define QSBR_QUIESCENT_STATE(tstate) +#endif + + +/* Do interpreter dispatch accounting for tracing and instrumentation */ +#define DISPATCH() \ + { \ + assert(frame->stackpointer == NULL); \ + NEXTOPARG(); \ + PRE_DISPATCH_GOTO(); \ + DISPATCH_GOTO(); \ + } + +#define DISPATCH_NON_TRACING() \ + { \ + assert(frame->stackpointer == NULL); \ + NEXTOPARG(); \ + PRE_DISPATCH_GOTO(); \ + DISPATCH_GOTO_NON_TRACING(); \ + } + +#define DISPATCH_SAME_OPARG() \ + { \ + opcode = next_instr->op.code; \ + PRE_DISPATCH_GOTO(); \ + DISPATCH_GOTO_NON_TRACING(); \ + } + +#define DISPATCH_INLINED(NEW_FRAME) \ + do { \ + assert(!IS_PEP523_HOOKED(tstate)); \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + assert((NEW_FRAME)->previous == frame); \ + frame = tstate->current_frame = (NEW_FRAME); \ + CALL_STAT_INC(inlined_py_calls); \ + JUMP_TO_LABEL(start_frame); \ + } while (0) + +/* Tuple access macros */ + +#ifndef Py_DEBUG +#define GETITEM(v, i) PyTuple_GET_ITEM((v), (i)) +#else +static inline PyObject * +GETITEM(PyObject *v, Py_ssize_t i) { + assert(PyTuple_Check(v)); + assert(i >= 0); + assert(i < PyTuple_GET_SIZE(v)); + return PyTuple_GET_ITEM(v, i); +} +#endif + +/* Code access macros */ + +/* The integer overflow is checked by an assertion below. */ +#define INSTR_OFFSET() ((int)(next_instr - _PyFrame_GetBytecode(frame))) +#define NEXTOPARG() do { \ + _Py_CODEUNIT word = {.cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t*)next_instr)}; \ + opcode = word.op.code; \ + oparg = word.op.arg; \ + } while (0) + +/* JUMPBY makes the generator identify the instruction as a jump. SKIP_OVER is + * for advancing to the next instruction, taking into account cache entries + * and skipped instructions. + */ +#define JUMPBY(x) (next_instr += (x)) +#define SKIP_OVER(x) (next_instr += (x)) + +#define STACK_LEVEL() ((int)(stack_pointer - _PyFrame_Stackbase(frame))) +#define STACK_SIZE() (_PyFrame_GetCode(frame)->co_stacksize) + +#define WITHIN_STACK_BOUNDS() \ + (frame->owner == FRAME_OWNED_BY_INTERPRETER || (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE())) + +#if defined(Py_DEBUG) && !defined(_Py_JIT) +// This allows temporary stack "overflows", provided it's all in the cache at any point of time. +#define WITHIN_STACK_BOUNDS_IGNORING_CACHE() \ + (frame->owner == FRAME_OWNED_BY_INTERPRETER || (STACK_LEVEL() >= 0 && (STACK_LEVEL()) <= STACK_SIZE())) +#else +#define WITHIN_STACK_BOUNDS_IGNORING_CACHE WITHIN_STACK_BOUNDS +#endif + +/* Data access macros */ +#define FRAME_CO_CONSTS (_PyFrame_GetCode(frame)->co_consts) +#define FRAME_CO_NAMES (_PyFrame_GetCode(frame)->co_names) + +/* Local variable macros */ + +#define LOCALS_ARRAY (frame->localsplus) +#define GETLOCAL(i) (frame->localsplus[i]) + + +#ifdef Py_STATS +#define UPDATE_MISS_STATS(INSTNAME) \ + do { \ + STAT_INC(opcode, miss); \ + STAT_INC((INSTNAME), miss); \ + /* The counter is always the first cache entry: */ \ + if (ADAPTIVE_COUNTER_TRIGGERS(next_instr->cache)) { \ + STAT_INC((INSTNAME), deopt); \ + } \ + } while (0) +#else +#define UPDATE_MISS_STATS(INSTNAME) ((void)0) +#endif + + +// Try to lock an object in the free threading build, if it's not already +// locked. Use with a DEOPT_IF() to deopt if the object is already locked. +// These are no-ops in the default GIL build. The general pattern is: +// +// DEOPT_IF(!LOCK_OBJECT(op)); +// if (/* condition fails */) { +// UNLOCK_OBJECT(op); +// DEOPT_IF(true); +// } +// ... +// UNLOCK_OBJECT(op); +// +// NOTE: The object must be unlocked on every exit code path and you should +// avoid any potentially escaping calls (like PyStackRef_CLOSE) while the +// object is locked. +#ifdef Py_GIL_DISABLED +# define LOCK_OBJECT(op) PyMutex_LockFast(&(_PyObject_CAST(op))->ob_mutex) +# define UNLOCK_OBJECT(op) PyMutex_Unlock(&(_PyObject_CAST(op))->ob_mutex) +#else +# define LOCK_OBJECT(op) (1) +# define UNLOCK_OBJECT(op) ((void)0) +#endif + +#define GLOBALS() frame->f_globals +#define BUILTINS() frame->f_builtins +#define LOCALS() frame->f_locals +#define CONSTS() _PyFrame_GetCode(frame)->co_consts +#define NAMES() _PyFrame_GetCode(frame)->co_names + +#if defined(WITH_DTRACE) && !defined(Py_BUILD_CORE_MODULE) +static void dtrace_function_entry(_PyInterpreterFrame *); +static void dtrace_function_return(_PyInterpreterFrame *); + +#define DTRACE_FUNCTION_ENTRY() \ + if (PyDTrace_FUNCTION_ENTRY_ENABLED()) { \ + dtrace_function_entry(frame); \ + } + +#define DTRACE_FUNCTION_RETURN() \ + if (PyDTrace_FUNCTION_RETURN_ENABLED()) { \ + dtrace_function_return(frame); \ + } +#else +#define DTRACE_FUNCTION_ENTRY() ((void)0) +#define DTRACE_FUNCTION_RETURN() ((void)0) +#endif + +/* This takes a uint16_t instead of a _Py_BackoffCounter, + * because it is used directly on the cache entry in generated code, + * which is always an integral type. */ +// Force re-specialization when tracing a side exit to get good side exits. +#define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \ + backoff_counter_triggers(forge_backoff_counter((COUNTER))) + +#define ADVANCE_ADAPTIVE_COUNTER(COUNTER) \ + do { \ + (COUNTER) = advance_backoff_counter((COUNTER)); \ + } while (0); + +#define PAUSE_ADAPTIVE_COUNTER(COUNTER) \ + do { \ + (COUNTER) = pause_backoff_counter((COUNTER)); \ + } while (0); + +#ifdef ENABLE_SPECIALIZATION +/* Multiple threads may execute these concurrently if thread-local bytecode is + * disabled and they all execute the main copy of the bytecode. Specialization + * is disabled in that case so the value is unused, but the RMW cycle should be + * free of data races. + */ +#define RECORD_BRANCH_TAKEN(bitset, flag) \ + FT_ATOMIC_STORE_UINT16_RELAXED( \ + bitset, (FT_ATOMIC_LOAD_UINT16_RELAXED(bitset) << 1) | (flag)) +#else +#define RECORD_BRANCH_TAKEN(bitset, flag) +#endif + +#define UNBOUNDLOCAL_ERROR_MSG \ + "cannot access local variable '%s' where it is not associated with a value" +#define UNBOUNDFREE_ERROR_MSG \ + "cannot access free variable '%s' where it is not associated with a value" \ + " in enclosing scope" +#define NAME_ERROR_MSG "name '%.200s' is not defined" + +// If a trace function sets a new f_lineno and +// *then* raises, we use the destination when searching +// for an exception handler, displaying the traceback, and so on +#define INSTRUMENTED_JUMP(src, dest, event) \ +do { \ + _Py_CODEUNIT *_dest = (dest); \ + if (tstate->tracing) {\ + next_instr = _dest; \ + } else { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + next_instr = _Py_call_instrumentation_jump(this_instr, tstate, event, frame, src, _dest); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + if (next_instr == NULL) { \ + next_instr = _dest + 1; \ + JUMP_TO_LABEL(error); \ + } \ + } \ +} while (0); + + +static inline int _Py_EnterRecursivePy(PyThreadState *tstate) { + return (tstate->py_recursion_remaining-- <= 0) && + _Py_CheckRecursiveCallPy(tstate); +} + +static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { + tstate->py_recursion_remaining++; +} + +/* Implementation of "macros" that modify the instruction pointer, + * stack pointer, or frame pointer. + * These need to treated differently by tier 1 and 2. + * The Tier 1 version is here; Tier 2 is inlined in ceval.c. */ + +#define LOAD_IP(OFFSET) do { \ + next_instr = frame->instr_ptr + (OFFSET); \ + } while (0) + +/* There's no STORE_IP(), it's inlined by the code generator. */ + +#define LOAD_SP() \ +stack_pointer = _PyFrame_GetStackPointer(frame) + +#define SAVE_SP() \ +_PyFrame_SetStackPointer(frame, stack_pointer) + +/* Tier-switching macros. */ + +#define TIER1_TO_TIER2(EXECUTOR) \ +do { \ + OPT_STAT_INC(traces_executed); \ + next_instr = _Py_jit_entry((EXECUTOR), frame, stack_pointer, tstate); \ + frame = tstate->current_frame; \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + int keep_tracing_bit = (uintptr_t)next_instr & 1; \ + next_instr = (_Py_CODEUNIT *)(((uintptr_t)next_instr) & (~1)); \ + if (next_instr == NULL) { \ + /* gh-140104: The exception handler expects frame->instr_ptr + to after this_instr, not this_instr! */ \ + next_instr = frame->instr_ptr + 1; \ + JUMP_TO_LABEL(error); \ + } \ + if (keep_tracing_bit) { \ + assert(uop_buffer_length(&((_PyThreadStateImpl *)tstate)->jit_tracer_state->code_buffer)); \ + ENTER_TRACING(); \ + DISPATCH_NON_TRACING(); \ + } \ + DISPATCH(); \ +} while (0) + +#define TIER2_TO_TIER2(EXECUTOR) \ +do { \ + OPT_STAT_INC(traces_executed); \ + current_executor = (EXECUTOR); \ + goto tier2_start; \ +} while (0) + +#define GOTO_TIER_ONE_SETUP \ + tstate->current_executor = NULL; \ + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \ + _PyFrame_SetStackPointer(frame, stack_pointer); + +#define GOTO_TIER_ONE(TARGET) \ + do \ + { \ + GOTO_TIER_ONE_SETUP \ + return (_Py_CODEUNIT *)(TARGET); \ + } while (0) + +#define GOTO_TIER_ONE_CONTINUE_TRACING(TARGET) \ + do \ + { \ + GOTO_TIER_ONE_SETUP \ + return (_Py_CODEUNIT *)(((uintptr_t)(TARGET))| 1); \ + } while (0) + +#define CURRENT_OPARG() (next_uop[-1].oparg) +#define CURRENT_OPERAND0_64() (next_uop[-1].operand0) +#define CURRENT_OPERAND1_64() (next_uop[-1].operand1) +#define CURRENT_OPERAND0_32() (next_uop[-1].operand0) +#define CURRENT_OPERAND1_32() (next_uop[-1].operand1) +#define CURRENT_OPERAND0_16() (next_uop[-1].operand0) +#define CURRENT_OPERAND1_16() (next_uop[-1].operand1) +#define CURRENT_TARGET() (next_uop[-1].target) + +#define JUMP_TO_JUMP_TARGET() goto jump_to_jump_target +#define JUMP_TO_ERROR() goto jump_to_error_target + +/* Stackref macros */ + +/* How much scratch space to give stackref to PyObject* conversion. */ +#define MAX_STACKREF_SCRATCH 10 + +#define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \ + /* +1 because vectorcall might use -1 to write self */ \ + PyObject *NAME##_temp[MAX_STACKREF_SCRATCH+1]; \ + PyObject **NAME = _PyObjectArray_FromStackRefArray(ARGS, ARG_COUNT, NAME##_temp); + +#define STACKREFS_TO_PYOBJECTS_CLEANUP(NAME) \ + /* +1 because we +1 previously */ \ + _PyObjectArray_Free(NAME - 1, NAME##_temp); + +#define CONVERSION_FAILED(NAME) ((NAME) == NULL) + +#if defined(Py_DEBUG) && !defined(_Py_JIT) +#define SET_CURRENT_CACHED_VALUES(N) current_cached_values = (N) +#define CHECK_CURRENT_CACHED_VALUES(N) assert(current_cached_values == (N)) +#else +#define SET_CURRENT_CACHED_VALUES(N) ((void)0) +#define CHECK_CURRENT_CACHED_VALUES(N) ((void)0) +#endif + +#define IS_PEP523_HOOKED(tstate) (tstate->interp->eval_frame != NULL) + +static inline int +check_periodics(PyThreadState *tstate) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + return _Py_HandlePending(tstate); + } + return 0; +} + +static inline int +check_periodics_at_end(PyThreadState *tstate, _PyInterpreterFrame *frame) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + // Do not handle pending interrupts if the previous instruction was LOAD_SPECIAL + // This may also not handle interrupts if a cache looks like LOAD_SPECIAL, + // but this is benign as we won't skip periodic checks indefinitely. + if (frame->instr_ptr[-1].op.code == LOAD_SPECIAL) { + return 0; + } + return _Py_HandlePending(tstate); + } + return 0; +} + +// Mark the generator as executing. Returns true if the state was changed, +// false if it was already executing or finished. +static inline bool +gen_try_set_executing(PyGenObject *gen) +{ +#ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { + int8_t frame_state = _Py_atomic_load_int8_relaxed(&gen->gi_frame_state); + while (frame_state < FRAME_SUSPENDED_YIELD_FROM_LOCKED) { + if (_Py_atomic_compare_exchange_int8(&gen->gi_frame_state, + &frame_state, + FRAME_EXECUTING)) { + return true; + } + } + // NB: We return false for FRAME_SUSPENDED_YIELD_FROM_LOCKED as well. + // That case is rare enough that we can just handle it in the deopt. + return false; + } +#endif + // Use faster non-atomic modifications in the GIL-enabled build and when + // the object is uniquely referenced in the free-threaded build. + if (gen->gi_frame_state < FRAME_EXECUTING) { + assert(gen->gi_frame_state != FRAME_SUSPENDED_YIELD_FROM_LOCKED); + gen->gi_frame_state = FRAME_EXECUTING; + return true; + } + return false; +} + +// Macro for inplace float binary ops (tier 2 only). +// Mutates the uniquely-referenced TARGET operand in place. +// TARGET must be either left or right. +#define FLOAT_INPLACE_OP(left, right, TARGET, OP) \ + do { \ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); \ + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); \ + assert(PyFloat_CheckExact(left_o)); \ + assert(PyFloat_CheckExact(right_o)); \ + assert(_PyObject_IsUniquelyReferenced( \ + PyStackRef_AsPyObjectBorrow(TARGET))); \ + STAT_INC(BINARY_OP, hit); \ + double _dres = \ + ((PyFloatObject *)left_o)->ob_fval \ + OP ((PyFloatObject *)right_o)->ob_fval; \ + ((PyFloatObject *)PyStackRef_AsPyObjectBorrow(TARGET)) \ + ->ob_fval = _dres; \ + } while (0) + +// Inplace float true division. Sets _divop_err to 1 on zero division. +// Caller must check _divop_err and call ERROR_NO_POP() if set. +#define FLOAT_INPLACE_DIVOP(left, right, TARGET) \ + int _divop_err = 0; \ + do { \ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); \ + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); \ + assert(PyFloat_CheckExact(left_o)); \ + assert(PyFloat_CheckExact(right_o)); \ + assert(_PyObject_IsUniquelyReferenced( \ + PyStackRef_AsPyObjectBorrow(TARGET))); \ + STAT_INC(BINARY_OP, hit); \ + double _divisor = ((PyFloatObject *)right_o)->ob_fval; \ + if (_divisor == 0.0) { \ + PyErr_SetString(PyExc_ZeroDivisionError, \ + "float division by zero"); \ + _divop_err = 1; \ + break; \ + } \ + double _dres = ((PyFloatObject *)left_o)->ob_fval / _divisor; \ + ((PyFloatObject *)PyStackRef_AsPyObjectBorrow(TARGET)) \ + ->ob_fval = _dres; \ + } while (0) + +// Inplace compact int operation. TARGET is expected to be uniquely +// referenced at the optimizer level, but at runtime it may be a +// cached small int singleton. We check _Py_IsImmortal on TARGET +// to decide whether inplace mutation is safe. +// +// After the macro, _int_inplace_res holds the result (may be NULL +// on allocation failure). On success, TARGET was mutated in place +// and _int_inplace_res is a DUP'd reference to it. On fallback +// (small int target, small int result, or overflow), _int_inplace_res +// is from FUNC (_PyCompactLong_Add etc.). +// FUNC is the fallback function (_PyCompactLong_Add etc.) +#define INT_INPLACE_OP(left, right, TARGET, OP, FUNC) \ + _PyStackRef _int_inplace_res = PyStackRef_NULL; \ + do { \ + PyObject *target_o = PyStackRef_AsPyObjectBorrow(TARGET); \ + if (_Py_IsImmortal(target_o)) { \ + break; \ + } \ + assert(_PyObject_IsUniquelyReferenced(target_o)); \ + Py_ssize_t left_val = _PyLong_CompactValue( \ + (PyLongObject *)PyStackRef_AsPyObjectBorrow(left)); \ + Py_ssize_t right_val = _PyLong_CompactValue( \ + (PyLongObject *)PyStackRef_AsPyObjectBorrow(right)); \ + Py_ssize_t result = left_val OP right_val; \ + if (!_PY_IS_SMALL_INT(result) \ + && ((twodigits)((stwodigits)result) + PyLong_MASK \ + < (twodigits)PyLong_MASK + PyLong_BASE)) \ + { \ + _PyLong_SetSignAndDigitCount( \ + (PyLongObject *)target_o, result < 0 ? -1 : 1, 1); \ + ((PyLongObject *)target_o)->long_value.ob_digit[0] = \ + (digit)(result < 0 ? -result : result); \ + _int_inplace_res = PyStackRef_DUP(TARGET); \ + break; \ + } \ + } while (0); \ + if (PyStackRef_IsNull(_int_inplace_res)) { \ + _int_inplace_res = FUNC( \ + (PyLongObject *)PyStackRef_AsPyObjectBorrow(left), \ + (PyLongObject *)PyStackRef_AsPyObjectBorrow(right)); \ + } + +#define CALL_TP_ITERITEM_NO_ESCAPE(ITER, INDEX) \ + Py_TYPE(ITER)->_tp_iteritem((ITER), (INDEX)) diff --git a/cinderx/Interpreter/3.15/Includes/generated_cases.c.h b/cinderx/Interpreter/3.15/Includes/generated_cases.c.h index 2a28fa113..5be704c5c 100644 --- a/cinderx/Interpreter/3.15/Includes/generated_cases.c.h +++ b/cinderx/Interpreter/3.15/Includes/generated_cases.c.h @@ -142,10 +142,11 @@ double dres = ((PyFloatObject *)left_o)->ob_fval + ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { JUMP_TO_LABEL(error); } + res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; } @@ -290,10 +291,10 @@ assert(PyUnicode_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = PyUnicode_Concat(left_o, right_o); - res = PyStackRef_FromPyObjectSteal(res_o); - if (PyStackRef_IsNull(res)) { + if (res_o == NULL) { JUMP_TO_LABEL(error); } + res = PyStackRef_FromPyObjectSteal(res_o); l = left; r = right; } @@ -342,11 +343,13 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); - assert(d && d->guard); + assert(d != NULL); _PyFrame_SetStackPointer(frame, stack_pointer); - int res = d->guard(left_o, right_o); + int match = (d->guard != NULL) + ? d->guard(left_o, right_o) + : (Py_TYPE(left_o) == d->lhs_type && Py_TYPE(right_o) == d->rhs_type); stack_pointer = _PyFrame_GetStackPointer(frame); - if (!res) { + if (!match) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -367,6 +370,9 @@ if (res_o == NULL) { JUMP_TO_LABEL(error); } + assert(d->result_type == NULL || Py_TYPE(res_o) == d->result_type); + assert(!d->result_unique || Py_REFCNT(res_o) == 1 || _Py_IsImmortal(res_o)); + assert(!PyFloat_CheckExact(res_o) || Py_REFCNT(res_o) == 1); res = PyStackRef_FromPyObjectSteal(res_o); l = left; r = right; @@ -522,10 +528,11 @@ double dres = ((PyFloatObject *)left_o)->ob_fval * ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { JUMP_TO_LABEL(error); } + res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; } @@ -639,11 +646,16 @@ _PyStackRef ds; _PyStackRef ss; _PyStackRef value; - // _GUARD_NOS_DICT + // _GUARD_NOS_DICT_SUBSCRIPT { nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { + if (!Py_TYPE(o)->tp_as_mapping) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (Py_TYPE(o)->tp_as_mapping->mp_subscript != _PyDict_Subscript) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -656,18 +668,12 @@ dict_st = nos; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); + assert(Py_TYPE(dict)->tp_as_mapping->mp_subscript == _PyDict_Subscript); STAT_INC(BINARY_OP, hit); - PyObject *res_o; _PyFrame_SetStackPointer(frame, stack_pointer); - int rc = PyDict_GetItemRef(dict, sub, &res_o); + PyObject *res_o = _PyDict_Subscript(dict, sub); stack_pointer = _PyFrame_GetStackPointer(frame); - if (rc == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetKeyError(sub); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - if (rc <= 0) { + if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); @@ -827,12 +833,10 @@ PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); assert(PyLong_CheckExact(sub)); assert(PyList_CheckExact(list)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + if (index < 0) { + index += PyList_GET_SIZE(list); } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; #ifdef Py_GIL_DISABLED _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); @@ -842,15 +846,13 @@ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - STAT_INC(BINARY_OP, hit); res = PyStackRef_FromPyObjectSteal(res_o); #else - if (index >= PyList_GET_SIZE(list)) { + if (index < 0 || index >= PyList_GET_SIZE(list)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - STAT_INC(BINARY_OP, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); res = PyStackRef_FromPyObjectNew(res_o); @@ -1276,10 +1278,11 @@ double dres = ((PyFloatObject *)left_o)->ob_fval - ((PyFloatObject *)right_o)->ob_fval; - res = PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(dres)); - if (PyStackRef_IsNull(res)) { + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { JUMP_TO_LABEL(error); } + res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; } @@ -1398,28 +1401,53 @@ stop = stack_pointer[-1]; start = stack_pointer[-2]; container = stack_pointer[-3]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); - stack_pointer = _PyFrame_GetStackPointer(frame); + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); PyObject *res_o; - if (slice == NULL) { - res_o = NULL; + if (PyList_CheckExact(container_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = _PyList_BinarySlice(container_o, start_o, stop_o); + stack_pointer = _PyFrame_GetStackPointer(frame); } - else { - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + else if (PyTuple_CheckExact(container_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); - res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); - Py_DECREF(slice); + res_o = _PyTuple_BinarySlice(container_o, start_o, stop_o); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; } - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + else if (PyUnicode_CheckExact(container_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = _PyUnicode_BinarySlice(container_o, start_o, stop_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + PyObject *slice = PySlice_New(start_o, stop_o, NULL); + if (slice == NULL) { + res_o = NULL; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = PyObject_GetItem(container_o, slice); + Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(container); + _PyStackRef tmp = stop; + stop = PyStackRef_NULL; + stack_pointer[-1] = stop; + PyStackRef_CLOSE(tmp); + tmp = start; + start = PyStackRef_NULL; + stack_pointer[-2] = start; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-3] = container; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -1839,7 +1867,7 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); + int err = check_periodics_at_end(tstate, frame); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { JUMP_TO_LABEL(error); @@ -1875,7 +1903,7 @@ JUMP_TO_PREDICTED(CALL); } } - // _CHECK_AND_ALLOCATE_OBJECT + // _CHECK_OBJECT { self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; @@ -1897,6 +1925,21 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _ALLOCATE_OBJECT + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(self_or_null)); + assert(PyType_Check(callable_o)); + PyTypeObject *tp = (PyTypeObject *)callable_o; assert(tp->tp_new == PyBaseObject_Type.tp_new); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); assert(tp->tp_alloc == PyType_GenericAlloc); @@ -2257,13 +2300,11 @@ _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; - _PyStackRef res; + _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - // _CALL_BUILTIN_CLASS + // _GUARD_CALLABLE_BUILTIN_CLASS { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyType_Check(callable_o)) { @@ -2272,38 +2313,59 @@ JUMP_TO_PREDICTED(CALL); } PyTypeObject *tp = (PyTypeObject *)callable_o; + if (tp->tp_vectorcall == NULL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_BUILTIN_CLASS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } - if (tp->tp_vectorcall == NULL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _Py_CallBuiltinClass_StackRefSteal( + PyObject *res_o = _Py_CallBuiltinClass_StackRef( callable, arguments, total_args); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } - res = PyStackRef_FromPyObjectSteal(res_o); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } - // _CHECK_PERIODIC_AT_END + // _POP_TOP_OPARG { - stack_pointer[-2 - oparg] = res; + args = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef_CloseStack(args, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = self_or_null; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _CHECK_PERIODIC_AT_END + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics_at_end(tstate, frame); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { JUMP_TO_LABEL(error); @@ -2326,20 +2388,12 @@ _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; - _PyStackRef res; + _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST + // _GUARD_CALLABLE_BUILTIN_FAST { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -2351,28 +2405,55 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CALL_BUILTIN_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _Py_BuiltinCallFast_StackRefSteal( + PyObject *res_o = _Py_BuiltinCallFast_StackRef( callable, arguments, total_args ); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } - res = PyStackRef_FromPyObjectSteal(res_o); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } - // _CHECK_PERIODIC_AT_END + // _POP_TOP_OPARG { - stack_pointer[-2 - oparg] = res; + args = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef_CloseStack(args, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = self_or_null; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _CHECK_PERIODIC_AT_END + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics_at_end(tstate, frame); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { JUMP_TO_LABEL(error); @@ -2395,20 +2476,12 @@ _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; - _PyStackRef res; + _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST_WITH_KEYWORDS + // _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); @@ -2420,24 +2493,51 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CALL_BUILTIN_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRefSteal(callable, arguments, total_args); + PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRef(callable, arguments, total_args); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } - res = PyStackRef_FromPyObjectSteal(res_o); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } - // _CHECK_PERIODIC_AT_END + // _POP_TOP_OPARG { - stack_pointer[-2 - oparg] = res; + args = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef_CloseStack(args, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = self_or_null; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _CHECK_PERIODIC_AT_END + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics_at_end(tstate, frame); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { JUMP_TO_LABEL(error); @@ -2466,37 +2566,46 @@ _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - // _CALL_BUILTIN_O + // _GUARD_CALLABLE_BUILTIN_O { - args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - if (total_args != 1) { + if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (!PyCFunction_CheckExact(callable_o)) { + if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args != 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CHECK_RECURSION_LIMIT + { if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CALL_BUILTIN_O + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyStackRef_IsNull(self_or_null)) { + args--; + } STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; @@ -2535,7 +2644,7 @@ // _CHECK_PERIODIC_AT_END { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); + int err = check_periodics_at_end(tstate, frame); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { JUMP_TO_LABEL(error); @@ -2641,7 +2750,7 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); + int err = check_periodics_at_end(tstate, frame); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { JUMP_TO_LABEL(error); @@ -2948,7 +3057,7 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); + int err = check_periodics_at_end(tstate, frame); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { JUMP_TO_LABEL(error); @@ -2967,23 +3076,28 @@ INSTRUCTION_STATS(CALL_INTRINSIC_1); _PyStackRef value; _PyStackRef res; - value = stack_pointer[-1]; - assert(oparg <= MAX_INTRINSIC_1); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - JUMP_TO_LABEL(error); + _PyStackRef v; + // _CALL_INTRINSIC_1 + { + value = stack_pointer[-1]; + assert(oparg <= MAX_INTRINSIC_1); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + v = value; + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _POP_TOP + { + value = v; + stack_pointer[-1] = res; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -2998,31 +3112,44 @@ _PyStackRef value2_st; _PyStackRef value1_st; _PyStackRef res; - value1_st = stack_pointer[-1]; - value2_st = stack_pointer[-2]; - assert(oparg <= MAX_INTRINSIC_2); - PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); - PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - _PyStackRef tmp = value1_st; - value1_st = PyStackRef_NULL; - stack_pointer[-1] = value1_st; - PyStackRef_CLOSE(tmp); - tmp = value2_st; - value2_st = PyStackRef_NULL; - stack_pointer[-2] = value2_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - if (res_o == NULL) { - JUMP_TO_LABEL(error); + _PyStackRef vs1; + _PyStackRef vs2; + _PyStackRef value; + // _CALL_INTRINSIC_2 + { + value1_st = stack_pointer[-1]; + value2_st = stack_pointer[-2]; + assert(oparg <= MAX_INTRINSIC_2); + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + vs1 = value1_st; + vs2 = value2_st; + } + // _POP_TOP + { + value = vs2; + stack_pointer[-2] = res; + stack_pointer[-1] = vs1; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = vs1; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -3418,7 +3545,7 @@ stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); + int err = check_periodics_at_end(tstate, frame); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { JUMP_TO_LABEL(error); @@ -3656,8 +3783,7 @@ // _GUARD_NOS_NOT_NULL { nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (o == NULL) { + if (PyStackRef_IsNull(nos)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -3730,69 +3856,95 @@ _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; - _PyStackRef res; + _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST + // _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST { args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (total_args == 0) { + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + if (method->d_method->ml_flags != METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_FASTCALL) { + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args == 0) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - assert(self != NULL); + PyObject *self = PyStackRef_AsPyObjectBorrow( + PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CALL_METHOD_DESCRIPTOR_FAST + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + assert(self != NULL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( + PyCFunctionFast cfunc = _PyCFunctionFast_CAST(method->d_method->ml_meth); + PyObject *res_o = _PyCallMethodDescriptorFast_StackRef( callable, - meth, + cfunc, self, arguments, total_args ); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } - res = PyStackRef_FromPyObjectSteal(res_o); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } - // _CHECK_PERIODIC_AT_END + // _POP_TOP_OPARG { - stack_pointer[-2 - oparg] = res; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef_CloseStack(args, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = self_or_null; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _CHECK_PERIODIC_AT_END + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics_at_end(tstate, frame); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { JUMP_TO_LABEL(error); @@ -3815,15 +3967,26 @@ _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; - _PyStackRef res; + _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + // _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS { args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (method->d_method->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { @@ -3835,50 +3998,65 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + } + // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; } - PyTypeObject *d_type = method->d_common.d_type; PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); assert(self != NULL); - if (!Py_IS_TYPE(self, d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords_CAST(method->d_method->ml_meth); + PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRef( callable, - meth, + cfunc, self, arguments, total_args ); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } - res = PyStackRef_FromPyObjectSteal(res_o); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); } - // _CHECK_PERIODIC_AT_END + // _POP_TOP_OPARG { - stack_pointer[-2 - oparg] = res; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef_CloseStack(args, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = self_or_null; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _CHECK_PERIODIC_AT_END + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics_at_end(tstate, frame); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { JUMP_TO_LABEL(error); @@ -3902,76 +4080,101 @@ _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef res; + _PyStackRef c; + _PyStackRef s; + _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_NOARGS + // _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS { args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - if (total_args != 1) { + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + if (method->d_method->ml_flags != METH_NOARGS) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDef *meth = method->d_method; - _PyStackRef self_stackref = args[0]; - PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - if (!Py_IS_TYPE(self, method->d_common.d_type)) { + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args != 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (meth->ml_flags != METH_NOARGS) { + PyObject *self = PyStackRef_AsPyObjectBorrow( + PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CHECK_RECURSION_LIMIT + { if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CALL_METHOD_DESCRIPTOR_NOARGS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + assert(oparg == 1 || !PyStackRef_IsNull(self_or_null)); + if (!PyStackRef_IsNull(self_or_null)) { + args--; + } + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; + PyCFunction cfunc = method->d_method->ml_meth; _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + c = callable; + s = args[0]; + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _POP_TOP + { + value = s; + stack_pointer[-2 - oparg] = res; + stack_pointer[-1 - oparg] = c; + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self_stackref); + PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; + } + // _POP_TOP + { + value = c; + stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); + PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC_AT_END { - stack_pointer[0] = res; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); + int err = check_periodics_at_end(tstate, frame); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { JUMP_TO_LABEL(error); @@ -4001,54 +4204,62 @@ _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_O + // _GUARD_CALLABLE_METHOD_DESCRIPTOR_O { args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (total_args != 2) { + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + if (method->d_method->ml_flags != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_O) { + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args != 2) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - if (_Py_ReachedRecursionLimit(tstate)) { + PyObject *self = PyStackRef_AsPyObjectBorrow( + PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } - _PyStackRef arg_stackref = arguments[1]; - _PyStackRef self_stackref = arguments[0]; - if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type)) { + } + // _CHECK_RECURSION_LIMIT + { + if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + } + // _CALL_METHOD_DESCRIPTOR_O + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + } STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; + PyCFunction cfunc = method->d_method->ml_meth; + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + PyObject *arg = PyStackRef_AsPyObjectBorrow(arguments[1]); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, - PyStackRef_AsPyObjectBorrow(self_stackref), - PyStackRef_AsPyObjectBorrow(arg_stackref)); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, arg); stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); @@ -4093,7 +4304,7 @@ // _CHECK_PERIODIC_AT_END { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); + int err = check_periodics_at_end(tstate, frame); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { JUMP_TO_LABEL(error); @@ -4168,7 +4379,7 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); + int err = check_periodics_at_end(tstate, frame); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { JUMP_TO_LABEL(error); @@ -4458,7 +4669,7 @@ // _CHECK_PERIODIC_AT_END { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); + int err = check_periodics_at_end(tstate, frame); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { JUMP_TO_LABEL(error); @@ -4533,7 +4744,7 @@ // _CHECK_PERIODIC_AT_END { _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); + int err = check_periodics_at_end(tstate, frame); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { JUMP_TO_LABEL(error); @@ -4721,13 +4932,16 @@ next_instr += 1; INSTRUCTION_STATS(CLEANUP_THROW); _PyStackRef sub_iter; + _PyStackRef null_in; _PyStackRef last_sent_val; _PyStackRef exc_value_st; _PyStackRef none; + _PyStackRef null_out; _PyStackRef value; exc_value_st = stack_pointer[-1]; last_sent_val = stack_pointer[-2]; - sub_iter = stack_pointer[-3]; + null_in = stack_pointer[-3]; + sub_iter = stack_pointer[-4]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); #if !_Py_TAIL_CALL_INTERP assert(throwflag); @@ -4741,7 +4955,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = sub_iter; sub_iter = value; - stack_pointer[-3] = sub_iter; + stack_pointer[-4] = sub_iter; PyStackRef_CLOSE(tmp); tmp = exc_value_st; exc_value_st = PyStackRef_NULL; @@ -4751,9 +4965,14 @@ last_sent_val = PyStackRef_NULL; stack_pointer[-2] = last_sent_val; PyStackRef_CLOSE(tmp); + tmp = null_in; + null_in = PyStackRef_NULL; + stack_pointer[-3] = null_in; + PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; + stack_pointer += -4; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + null_out = null_in; none = PyStackRef_None; } else { @@ -4763,8 +4982,9 @@ JUMP_TO_LABEL(exception_unwind); } stack_pointer[0] = none; - stack_pointer[1] = value; - stack_pointer += 2; + stack_pointer[1] = null_out; + stack_pointer[2] = value; + stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5145,11 +5365,11 @@ _PyStackRef l; _PyStackRef r; _PyStackRef value; - // _GUARD_TOS_DICT + // _GUARD_TOS_ANY_DICT { tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyDict_CheckExact(o)) { + if (!PyAnyDict_CheckExact(o)) { UPDATE_MISS_STATS(CONTAINS_OP); assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); JUMP_TO_PREDICTED(CONTAINS_OP); @@ -5162,7 +5382,7 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyDict_CheckExact(right_o)); + assert(PyAnyDict_CheckExact(right_o)); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); @@ -5507,31 +5727,38 @@ _PyStackRef callable; _PyStackRef dict; _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - callable = stack_pointer[-5 - (oparg - 1)]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_MergeEx(dict_o, update_o, 2); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { + _PyStackRef u; + _PyStackRef value; + // _DICT_MERGE + { + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + PyObject *dupkey = NULL; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatKwargsError(tstate, callable_o, update_o); + int err = _PyDict_MergeUniq(dict_o, update_o, &dupkey); stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatKwargsError(tstate, callable_o, update_o, dupkey); + Py_XDECREF(dupkey); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + u = update; + } + // _POP_TOP + { + value = u; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); + PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); } - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -5545,36 +5772,51 @@ INSTRUCTION_STATS(DICT_UPDATE); _PyStackRef dict; _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Update(dict_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { + _PyStackRef upd; + _PyStackRef value; + // _DICT_UPDATE + { + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + int err = PyDict_Update(dict_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *exc = _PyErr_GetRaisedException(tstate); + int has_keys = PyObject_HasAttrWithError(update_o, &_Py_ID(keys)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (has_keys == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'%T' object is not a mapping", + update_o); + Py_DECREF(exc); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_ChainExceptions1(exc); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + JUMP_TO_LABEL(error); } + upd = update; + } + // _POP_TOP + { + value = upd; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); + PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); } - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -5649,13 +5891,16 @@ next_instr += 1; INSTRUCTION_STATS(END_SEND); _PyStackRef receiver; + _PyStackRef index_or_null; _PyStackRef value; _PyStackRef val; value = stack_pointer[-1]; - receiver = stack_pointer[-2]; + index_or_null = stack_pointer[-2]; + receiver = stack_pointer[-3]; val = value; - stack_pointer[-2] = val; - stack_pointer += -1; + (void)index_or_null; + stack_pointer[-3] = val; + stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(receiver); @@ -5675,17 +5920,28 @@ INSTRUCTION_STATS(ENTER_EXECUTOR); opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; if (IS_JIT_TRACING()) { + int og_opcode = executor->vm_data.opcode; + int og_oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; + if (_PyJit_EnterExecutorShouldStopTracing(og_opcode)) { + if (_PyOpcode_Caches[_PyOpcode_Deopt[og_opcode]]) { + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + opcode = og_opcode; + oparg = og_oparg; + DISPATCH_GOTO_NON_TRACING(); + } JUMP_TO_LABEL(stop_tracing); } - PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); assert(executor->vm_data.code == code); assert(executor->vm_data.valid); assert(tstate->current_executor == NULL); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + uintptr_t iversion = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) != iversion) { opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; @@ -7544,52 +7800,103 @@ TARGET(FOR_ITER_TUPLE) { #if _Py_TAIL_CALL_INTERP - int opcode = FOR_ITER_TUPLE; + int opcode = FOR_ITER_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_TUPLE + { + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyTuple_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + } + // _ITER_JUMP_TUPLE + { + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + (void)tuple_o; + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + STAT_INC(FOR_ITER, hit); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + DISPATCH(); + } + } + // _ITER_NEXT_TUPLE + { + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + uintptr_t i = PyStackRef_UntagInt(null_or_index); + assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + } + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(FOR_ITER_VIRTUAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_VIRTUAL; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_TUPLE); + INSTRUCTION_STATS(FOR_ITER_VIRTUAL); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; _PyStackRef null_or_index; + _PyStackRef iter; _PyStackRef next; /* Skip 1 cache entry */ - // _ITER_CHECK_TUPLE + // _GUARD_TOS_NOT_NULL { null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyTuple_Type) { + if (PyStackRef_IsNull(null_or_index)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } - assert(PyStackRef_IsTaggedInt(null_or_index)); } - // _ITER_JUMP_TUPLE + // _FOR_ITER_VIRTUAL { - PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); - (void)tuple_o; - assert(Py_TYPE(tuple_o) == &PyTuple_Type); - STAT_INC(FOR_ITER, hit); - if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { - null_or_index = PyStackRef_TagInt(-1); + iter = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + Py_ssize_t index = PyStackRef_UntagInt(null_or_index); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyObject *next_o = next_index.object; + index = next_index.index; + if (next_o == NULL) { + if (index < 0) { + JUMP_TO_LABEL(error); + } JUMPBY(oparg + 1); - stack_pointer[-1] = null_or_index; DISPATCH(); } - } - // _ITER_NEXT_TUPLE - { - PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(tuple_o) == &PyTuple_Type); - uintptr_t i = PyStackRef_UntagInt(null_or_index); - assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); - null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + null_or_index = PyStackRef_TagInt(index); + next = PyStackRef_FromPyObjectSteal(next_o); } stack_pointer[-1] = null_or_index; stack_pointer[0] = next; @@ -7718,37 +8025,40 @@ (void)(opcode); #endif frame->instr_ptr = next_instr; - next_instr += 1; + next_instr += 2; INSTRUCTION_STATS(GET_ITER); + PREDICTED_GET_ITER:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; _PyStackRef iterable; _PyStackRef iter; _PyStackRef index_or_null; - iterable = stack_pointer[-1]; - #ifdef Py_STATS - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_GatherStats_GetIter(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - #endif - PyTypeObject *tp = PyStackRef_TYPE(iterable); - if (tp == &PyTuple_Type || tp == &PyList_Type) { - iter = iterable; - index_or_null = PyStackRef_TagInt(0); + // _SPECIALIZE_GET_ITER + { + iterable = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_GetIter(iterable, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(GET_ITER); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + // _GET_ITER + { _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); + _PyStackRef result = _PyEval_GetIter(iterable, &index_or_null, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - JUMP_TO_LABEL(error); + if (PyStackRef_IsError(result)) { + JUMP_TO_LABEL(pop_1_error); } - iter = PyStackRef_FromPyObjectSteal(iter_o); - index_or_null = PyStackRef_NULL; - stack_pointer += 1; + iter = result; } stack_pointer[-1] = iter; stack_pointer[0] = index_or_null; @@ -7757,6 +8067,76 @@ DISPATCH(); } + TARGET(GET_ITER_SELF) { + #if _Py_TAIL_CALL_INTERP + int opcode = GET_ITER_SELF; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(GET_ITER_SELF); + static_assert(INLINE_CACHE_ENTRIES_GET_ITER == 1, "incorrect cache size"); + _PyStackRef iterable; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_ITERATOR + { + iterable = stack_pointer[-1]; + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable)); + if (tp->tp_iter != PyObject_SelfIter) { + UPDATE_MISS_STATS(GET_ITER); + assert(_PyOpcode_Deopt[opcode] == (GET_ITER)); + JUMP_TO_PREDICTED(GET_ITER); + } + STAT_INC(GET_ITER, hit); + } + // _PUSH_NULL + { + res = PyStackRef_NULL; + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(GET_ITER_VIRTUAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = GET_ITER_VIRTUAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(GET_ITER_VIRTUAL); + static_assert(INLINE_CACHE_ENTRIES_GET_ITER == 1, "incorrect cache size"); + _PyStackRef iterable; + _PyStackRef zero; + /* Skip 1 cache entry */ + // _GUARD_ITER_VIRTUAL + { + iterable = stack_pointer[-1]; + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable)); + if (tp->_tp_iteritem == NULL) { + UPDATE_MISS_STATS(GET_ITER); + assert(_PyOpcode_Deopt[opcode] == (GET_ITER)); + JUMP_TO_PREDICTED(GET_ITER); + } + STAT_INC(GET_ITER, hit); + } + // _PUSH_TAGGED_ZERO + { + zero = PyStackRef_TagInt(0); + } + stack_pointer[0] = zero; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + TARGET(GET_LEN) { #if _Py_TAIL_CALL_INTERP int opcode = GET_LEN; @@ -7785,51 +8165,6 @@ DISPATCH(); } - TARGET(GET_YIELD_FROM_ITER) { - #if _Py_TAIL_CALL_INTERP - int opcode = GET_YIELD_FROM_ITER; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_YIELD_FROM_ITER); - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); - if (PyCoro_CheckExact(iterable_o)) { - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "in a non-coroutine generator"); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); - } - iter = iterable; - } - else if (PyGen_CheckExact(iterable_o)) { - iter = iterable; - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(iterable_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - JUMP_TO_LABEL(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = iterable; - iterable = iter; - stack_pointer[-1] = iterable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer[-1] = iter; - DISPATCH(); - } - TARGET(IMPORT_FROM) { #if _Py_TAIL_CALL_INTERP int opcode = IMPORT_FROM; @@ -8040,7 +8375,7 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); + int err = check_periodics_at_end(tstate, frame); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { JUMP_TO_LABEL(error); @@ -8208,7 +8543,7 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); + int err = check_periodics_at_end(tstate, frame); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { JUMP_TO_LABEL(error); @@ -8433,10 +8768,12 @@ next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_SEND); _PyStackRef receiver; + _PyStackRef index_or_null; _PyStackRef value; _PyStackRef val; value = stack_pointer[-1]; - receiver = stack_pointer[-2]; + index_or_null = stack_pointer[-2]; + receiver = stack_pointer[-3]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -8447,8 +8784,9 @@ } } val = value; - stack_pointer[-2] = val; - stack_pointer += -1; + (void)index_or_null; + stack_pointer[-3] = val; + stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(receiver); @@ -8880,8 +9218,9 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 1; + next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_RESUME); + /* Skip 1 cache entry */ // _LOAD_BYTECODE { #ifdef Py_GIL_DISABLED @@ -8963,6 +9302,7 @@ next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); _PyStackRef val; + _PyStackRef value; _PyStackRef retval; _PyStackRef res; // _RETURN_VALUE_EVENT @@ -8977,19 +9317,24 @@ JUMP_TO_LABEL(error); } } + // _MAKE_HEAP_SAFE + { + value = val; + value = PyStackRef_MakeHeapSafe(value); + } // _RETURN_VALUE { - retval = val; + retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); + _PyStackRef temp = retval; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(STACK_LEVEL() == 0); + DTRACE_FUNCTION_RETURN(); _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame* dying = frame; + _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; - CI_SET_ADAPTIVE_INTERPRETER_ENABLED_STATE _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(frame->return_offset); @@ -9012,9 +9357,10 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + opcode = INSTRUMENTED_YIELD_VALUE; _PyStackRef val; - _PyStackRef retval; _PyStackRef value; + _PyStackRef retval; // _YIELD_VALUE_EVENT { val = stack_pointer[-1]; @@ -9031,40 +9377,45 @@ DISPATCH(); } } + // _MAKE_HEAP_SAFE + { + value = val; + value = PyStackRef_MakeHeapSafe(value); + } // _YIELD_VALUE { - retval = val; + retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); frame->instr_ptr++; - PyGenObject* gen = _PyGen_GetGeneratorFromFrame(frame); + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + DTRACE_FUNCTION_RETURN(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame* gen_frame = frame; + _PyInterpreterFrame *gen_frame = frame; + _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; - CI_SET_ADAPTIVE_INTERPRETER_ENABLED_STATE gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert( - frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #if TIER_ONE && defined(Py_DEBUG) + if (!PyStackRef_IsNone(frame->f_executable)) { + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; + assert(opcode == SEND || opcode == FOR_ITER); + } #endif stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - value = PyStackRef_MakeHeapSafe(temp); + value = temp; LLTRACE_RESUME_FRAME(); } stack_pointer[0] = value; @@ -9221,16 +9572,19 @@ // _JIT { #ifdef _Py_TIER2 + bool is_resume = this_instr->op.code == RESUME_CHECK_JIT; _Py_BackoffCounter counter = this_instr[1].counter; - if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && - this_instr->op.code == JUMP_BACKWARD_JIT && + if ((backoff_counter_triggers(counter) && + !IS_JIT_TRACING() && + (this_instr->op.code == JUMP_BACKWARD_JIT || is_resume)) && next_instr->op.code != ENTER_EXECUTOR) { _Py_CODEUNIT *insert_exec_at = this_instr; while (oparg > 255) { oparg >>= 8; insert_exec_at--; } - int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, stack_pointer, 0, NULL, oparg, NULL); + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, + is_resume ? insert_exec_at : next_instr, stack_pointer, 0, NULL, oparg, NULL); if (succ) { ENTER_TRACING(); } @@ -9336,40 +9690,43 @@ INSTRUCTION_STATS(LIST_EXTEND); _PyStackRef list_st; _PyStackRef iterable_st; - iterable_st = stack_pointer[-1]; - list_st = stack_pointer[-2 - (oparg-1)]; - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (none_val == NULL) { + _PyStackRef i; + _PyStackRef value; + // _LIST_EXTEND + { + iterable_st = stack_pointer[-1]; + list_st = stack_pointer[-2 - (oparg-1)]; + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) - { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Clear(tstate); - _PyErr_Format(tstate, PyExc_TypeError, + if (none_val == NULL) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + if (matches && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + JUMP_TO_LABEL(error); } + assert(Py_IsNone(none_val)); + i = iterable_st; + } + // _POP_TOP + { + value = i; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable_st); + PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); } - assert(Py_IsNone(none_val)); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable_st); - stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH(); } @@ -9541,89 +9898,121 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } - // _LOAD_ATTR_CLASS + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME + { + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *getattribute = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)getattribute; + assert(func_version != 0); + if (f->func_version != func_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 2); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + STAT_INC(LOAD_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked( + tstate, PyStackRef_FromPyObjectNew(f), 2, frame); + pushed_frame->localsplus[0] = owner; + pushed_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = PyStackRef_FromPyObjectNew(descr); + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = owner; - owner = attr; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + CI_UPDATE_CALL_COUNT + LLTRACE_RESUME_FRAME(); } - stack_pointer += (oparg & 1); - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } - TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - /* Skip 1 cache entry */ - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *getattribute = read_obj(&this_instr[6].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert((oparg & 1) == 0); - if (IS_PEP523_HOOKED(tstate)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - PyTypeObject *cls = Py_TYPE(owner_o); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)getattribute; - assert(func_version != 0); - if (f->func_version != func_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 2); - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( - tstate, PyStackRef_FromPyObjectNew(f), 2, frame); - new_frame->localsplus[0] = owner; - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); - frame->return_offset = 10u ; - DISPATCH_INLINED(new_frame); - } - TARGET(LOAD_ATTR_INSTANCE_VALUE) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_INSTANCE_VALUE; @@ -9851,18 +10240,7 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - } + /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_WITH_VALUES { PyObject *descr = read_obj(&this_instr[6].cache); @@ -9919,7 +10297,7 @@ assert(keys->dk_kind == DICT_KEYS_UNICODE); assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_CONSUME(ep->me_value); if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); @@ -10045,18 +10423,7 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - } + /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { PyObject *descr = read_obj(&this_instr[6].cache); @@ -10110,29 +10477,19 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } - /* Skip 2 cache entries */ // _LOAD_ATTR_PROPERTY_FRAME { + uint32_t func_version = read_u32(&this_instr[4].cache); PyObject *fget = read_obj(&this_instr[6].cache); assert((oparg & 1) == 0); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; - PyCodeObject *code = (PyCodeObject *)f->func_code; - if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - if (code->co_kwonlyargcount) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - if (code->co_argcount != 1) { + if (f->func_version != func_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } + PyCodeObject *code = (PyCodeObject *)f->func_code; if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); @@ -10402,7 +10759,7 @@ INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); _PyStackRef value; assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + value = PyStackRef_FromPyObjectNew(Ci_common_consts[oparg]); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -10816,7 +11173,7 @@ } assert(keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_CONSUME(entries[index].me_value); if (res_o == NULL) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); @@ -10883,7 +11240,7 @@ assert(keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); assert(index < DK_SIZE(keys)); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_CONSUME(entries[index].me_value); if (res_o == NULL) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); @@ -11268,64 +11625,71 @@ _PyStackRef attr; _PyStackRef self_or_null; /* Skip 1 cache entry */ - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(oparg & 1); - if (global_super != (PyObject *)&PySuper_Type) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); - } - if (!PyType_Check(class)) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); - } - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; - PyObject *attr_o; + // _GUARD_LOAD_SUPER_ATTR_METHOD { - int *method_found_ptr = &method_found; - _PyFrame_SetStackPointer(frame, stack_pointer); - attr_o = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - if (attr_o == NULL) { - JUMP_TO_LABEL(error); + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + assert(oparg & 1); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } } - if (method_found) { - self_or_null = self_st; - } else { + // _LOAD_SUPER_ATTR_METHOD + { + self_st = stack_pointer[-1]; + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyTypeObject *cls = (PyTypeObject *)class; + int method_found = 0; + PyObject *attr_o; + { + int *method_found_ptr = &method_found; + _PyFrame_SetStackPointer(frame, stack_pointer); + attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + if (attr_o == NULL) { + JUMP_TO_LABEL(error); + } + if (method_found) { + self_or_null = self_st; + } else { + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + self_or_null = PyStackRef_NULL; + stack_pointer += 1; + } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self_st); + _PyStackRef tmp = global_super_st; + global_super_st = self_or_null; + stack_pointer[-2] = global_super_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-1] = class_st; + PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); - self_or_null = PyStackRef_NULL; - stack_pointer += 1; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + attr = PyStackRef_FromPyObjectSteal(attr_o); } - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = global_super_st; - global_super_st = self_or_null; - stack_pointer[-2] = global_super_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-1] = class_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[0] = attr; stack_pointer[1] = self_or_null; stack_pointer += 2; @@ -11364,26 +11728,32 @@ INSTRUCTION_STATS(MAKE_FUNCTION); _PyStackRef codeobj_st; _PyStackRef func; - codeobj_st = stack_pointer[-1]; - PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(codeobj_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (func_obj == NULL) { - JUMP_TO_LABEL(error); - } - _PyFunction_SetVersion( + _PyStackRef co; + _PyStackRef value; + // _MAKE_FUNCTION + { + codeobj_st = stack_pointer[-1]; + PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (func_obj == NULL) { + JUMP_TO_LABEL(error); + } + co = codeobj_st; + _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); - func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); - stack_pointer[0] = func; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); + } + // _POP_TOP + { + value = co; + stack_pointer[-1] = func; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } DISPATCH(); } @@ -11436,43 +11806,64 @@ _PyStackRef type; _PyStackRef names; _PyStackRef attrs; - names = stack_pointer[-1]; - type = stack_pointer[-2]; - subject = stack_pointer[-3]; - assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attrs_o = _PyEval_MatchClass(tstate, - PyStackRef_AsPyObjectBorrow(subject), - PyStackRef_AsPyObjectBorrow(type), oparg, - PyStackRef_AsPyObjectBorrow(names)); - _PyStackRef tmp = names; - names = PyStackRef_NULL; - stack_pointer[-1] = names; - PyStackRef_CLOSE(tmp); - tmp = type; - type = PyStackRef_NULL; - stack_pointer[-2] = type; - PyStackRef_CLOSE(tmp); - tmp = subject; - subject = PyStackRef_NULL; - stack_pointer[-3] = subject; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - if (attrs_o) { - assert(PyTuple_CheckExact(attrs_o)); - attrs = PyStackRef_FromPyObjectSteal(attrs_o); - } - else { - if (_PyErr_Occurred(tstate)) { - JUMP_TO_LABEL(error); + _PyStackRef s; + _PyStackRef tp; + _PyStackRef n; + _PyStackRef value; + // _MATCH_CLASS + { + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); + attrs = PyStackRef_FromPyObjectSteal(attrs_o); + } + else { + if (_PyErr_Occurred(tstate)) { + JUMP_TO_LABEL(error); + } + attrs = PyStackRef_None; } - attrs = PyStackRef_None; + s = subject; + tp = type; + n = names; + } + // _POP_TOP + { + value = n; + stack_pointer[-3] = attrs; + stack_pointer[-2] = s; + stack_pointer[-1] = tp; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = tp; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = s; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); } - stack_pointer[0] = attrs; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -11874,10 +12265,10 @@ (void)(opcode); #endif frame->instr_ptr = next_instr; - next_instr += 1; + next_instr += 2; INSTRUCTION_STATS(RESUME); PREDICTED_RESUME:; - _Py_CODEUNIT* const this_instr = next_instr - 1; + _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; // _LOAD_BYTECODE { @@ -11924,11 +12315,11 @@ } // _QUICKEN_RESUME { - #if ENABLE_SPECIALIZATION - if (tstate->tracing == 0 && this_instr->op.code == RESUME) { - FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); - } - #endif /* ENABLE_SPECIALIZATION */ + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Resume(this_instr, tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); } // _CHECK_PERIODIC_IF_NOT_YIELD_FROM { @@ -11952,9 +12343,10 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 1; + next_instr += 2; INSTRUCTION_STATS(RESUME_CHECK); - static_assert(0 == 0, "incorrect cache size"); + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { UPDATE_MISS_STATS(RESUME); @@ -11978,7 +12370,77 @@ assert(_PyOpcode_Deopt[opcode] == (RESUME)); JUMP_TO_PREDICTED(RESUME); } - #endif + #endif + DISPATCH(); + } + + TARGET(RESUME_CHECK_JIT) { + #if _Py_TAIL_CALL_INTERP + int opcode = RESUME_CHECK_JIT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(RESUME_CHECK_JIT); + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ + // _RESUME_CHECK + { + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + if (eval_breaker != version) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } + #endif + } + // _JIT + { + #ifdef _Py_TIER2 + bool is_resume = this_instr->op.code == RESUME_CHECK_JIT; + _Py_BackoffCounter counter = this_instr[1].counter; + if ((backoff_counter_triggers(counter) && + !IS_JIT_TRACING() && + (this_instr->op.code == JUMP_BACKWARD_JIT || is_resume)) && + next_instr->op.code != ENTER_EXECUTOR) { + _Py_CODEUNIT *insert_exec_at = this_instr; + while (oparg > 255) { + oparg >>= 8; + insert_exec_at--; + } + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, + is_resume ? insert_exec_at : next_instr, stack_pointer, 0, NULL, oparg, NULL); + if (succ) { + ENTER_TRACING(); + } + else { + this_instr[1].counter = restart_backoff_counter(counter); + } + } + else { + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + #endif + } DISPATCH(); } @@ -12031,24 +12493,33 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_VALUE); + _PyStackRef value; _PyStackRef retval; _PyStackRef res; - retval = stack_pointer[-1]; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(STACK_LEVEL() == 0); - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame* dying = frame; - frame = tstate->current_frame = dying->previous; - CI_SET_ADAPTIVE_INTERPRETER_ENABLED_STATE - _PyEval_FrameClearAndPop(tstate, dying); - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(frame->return_offset); - res = temp; - LLTRACE_RESUME_FRAME(); + // _MAKE_HEAP_SAFE + { + value = stack_pointer[-1]; + value = PyStackRef_MakeHeapSafe(value); + } + // _RETURN_VALUE + { + retval = value; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyStackRef temp = retval; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(STACK_LEVEL() == 0); + DTRACE_FUNCTION_RETURN(); + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(frame->return_offset); + res = temp; + LLTRACE_RESUME_FRAME(); + } stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -12067,11 +12538,12 @@ _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef receiver; + _PyStackRef null_or_index; _PyStackRef v; _PyStackRef retval; // _SPECIALIZE_SEND { - receiver = stack_pointer[-2]; + receiver = stack_pointer[-3]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION @@ -12089,8 +12561,8 @@ // _SEND { v = stack_pointer[-1]; + null_or_index = stack_pointer[-2]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - PyObject *retval_o; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if (!IS_PEP523_HOOKED(tstate) && (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && @@ -12109,50 +12581,101 @@ gen_frame->previous = frame; DISPATCH_INLINED(gen_frame); } - if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { + if (!PyStackRef_IsNull(null_or_index) && PyStackRef_IsNone(v)) { _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, receiver, &null_or_index); stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + JUMP_TO_LABEL(error); + } + JUMPBY(oparg); + stack_pointer[-2] = null_or_index; + DISPATCH(); + } + retval = item; } else { + PyObject *v_o = PyStackRef_AsPyObjectBorrow(v); _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = PyObject_CallMethodOneArg(receiver_o, - &_Py_ID(send), - PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - if (retval_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + PySendResultPair res = _PyIter_Send(receiver_o, v_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + if (res.kind == PYGEN_ERROR) { + JUMP_TO_LABEL(error); } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyGen_FetchStopIterationValue(&retval_o); + PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err == 0) { - assert(retval_o != NULL); + retval = PyStackRef_FromPyObjectSteal(res.object); + if (res.kind == PYGEN_RETURN) { JUMPBY(oparg); } - else { - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); - } + stack_pointer += 1; + } + } + stack_pointer[-2] = null_or_index; + stack_pointer[-1] = retval; + DISPATCH(); + } + + TARGET(SEND_ASYNC_GEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = SEND_ASYNC_GEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_ASYNC_GEN); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef null_in; + _PyStackRef v; + _PyStackRef asend; + _PyStackRef null_out; + _PyStackRef retval; + /* Skip 1 cache entry */ + // _GUARD_3OS_ASYNC_GEN_ASEND + { + iter = stack_pointer[-3]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (!PyAsyncGenASend_CheckExact(iter_o)) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + } + // _SEND_ASYNC_GEN + { + v = stack_pointer[-1]; + null_in = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyAsyncGenASend_CheckExact(iter_o)); + PyObject *val = PyStackRef_AsPyObjectBorrow(v); + PyObject *retval_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + PySendResult what = _PyAsyncGenASend_Send(iter_o, val, &retval_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (what == PYGEN_ERROR) { + JUMP_TO_LABEL(error); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); + asend = iter; + null_out = null_in; retval = PyStackRef_FromPyObjectSteal(retval_o); + if (what == PYGEN_RETURN) { + JUMPBY(oparg); + } } + stack_pointer[-2] = asend; + stack_pointer[-1] = null_out; stack_pointer[0] = retval; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -12186,7 +12709,7 @@ // _SEND_GEN_FRAME { v = stack_pointer[-1]; - receiver = stack_pointer[-2]; + receiver = stack_pointer[-3]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { UPDATE_MISS_STATS(SEND); @@ -12228,6 +12751,70 @@ DISPATCH(); } + TARGET(SEND_VIRTUAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = SEND_VIRTUAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_VIRTUAL); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef val; + _PyStackRef nos; + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef none; + _PyStackRef next; + /* Skip 1 cache entry */ + // _GUARD_TOS_IS_NONE + { + val = stack_pointer[-1]; + if (!PyStackRef_IsNone(val)) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + } + // _GUARD_NOS_NOT_NULL + { + nos = stack_pointer[-2]; + if (PyStackRef_IsNull(nos)) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + } + // _SEND_VIRTUAL + { + none = val; + null_or_index = nos; + iter = stack_pointer[-3]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + Py_ssize_t index = PyStackRef_UntagInt(null_or_index); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyObject *next_o = next_index.object; + index = next_index.index; + if (next_o == NULL) { + if (index < 0) { + JUMP_TO_LABEL(error); + } + next = none; + JUMPBY(oparg); + DISPATCH(); + } + next = PyStackRef_FromPyObjectSteal(next_o); + null_or_index = PyStackRef_TagInt(index); + } + stack_pointer[-2] = null_or_index; + stack_pointer[-1] = next; + DISPATCH(); + } + TARGET(SETUP_ANNOTATIONS) { #if _Py_TAIL_CALL_INTERP int opcode = SETUP_ANNOTATIONS; @@ -12286,8 +12873,21 @@ _PyStackRef v; v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; + PyObject *set_o = PyStackRef_AsPyObjectBorrow(set); + if (!PySet_CheckExact(set_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'%T' object is not a set", set_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), + int err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(v)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { @@ -12336,19 +12936,29 @@ INSTRUCTION_STATS(SET_UPDATE); _PyStackRef set; _PyStackRef iterable; - iterable = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + _PyStackRef i; + _PyStackRef value; + // _SET_UPDATE + { + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_LABEL(error); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + i = iterable; + } + // _POP_TOP + { + value = i; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); } DISPATCH(); } @@ -12421,21 +13031,25 @@ next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; _PyStackRef value; + _PyStackRef owner; _PyStackRef o; /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION_AND_LOCK + // _LOCK_OBJECT { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(type_version != 0); - if (!LOCK_OBJECT(owner_o)) { + value = stack_pointer[-1]; + if (!LOCK_OBJECT(PyStackRef_AsPyObjectBorrow(value))) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); JUMP_TO_PREDICTED(STORE_ATTR); } + } + // _GUARD_TYPE_VERSION_LOCKED + { + owner = value; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); PyTypeObject *tp = Py_TYPE(owner_o); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UNLOCK_OBJECT(owner_o); @@ -12966,11 +13580,16 @@ _PyStackRef dict_st; _PyStackRef sub; _PyStackRef st; - // _GUARD_NOS_DICT + // _GUARD_NOS_DICT_STORE_SUBSCRIPT { nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { + if (!Py_TYPE(o)->tp_as_mapping) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + if (Py_TYPE(o)->tp_as_mapping->mp_ass_subscript != _PyDict_StoreSubscript) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); JUMP_TO_PREDICTED(STORE_SUBSCR); @@ -12983,7 +13602,7 @@ dict_st = nos; value = stack_pointer[-3]; PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); + assert(Py_TYPE(dict)->tp_as_mapping->mp_ass_subscript == _PyDict_StoreSubscript); STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, @@ -13059,18 +13678,17 @@ PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); assert(PyLong_CheckExact(sub)); assert(PyList_CheckExact(list)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(STORE_SUBSCR); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); if (!LOCK_OBJECT(list)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); JUMP_TO_PREDICTED(STORE_SUBSCR); } - if (index >= PyList_GET_SIZE(list)) { + Py_ssize_t len = PyList_GET_SIZE(list); + if (index < 0) { + index += len; + } + if (index < 0 || index >= len) { UNLOCK_OBJECT(list); if (true) { UPDATE_MISS_STATS(STORE_SUBSCR); @@ -13454,9 +14072,12 @@ } DISPATCH(); } - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(tracer->prev_state.recorded_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + for (int i = 0; i < tracer->prev_state.recorded_count; i++) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(tracer->prev_state.recorded_values[i]); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + tracer->prev_state.recorded_count = 0; tracer->prev_state.instr = next_instr; PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); if (tracer->prev_state.instr_code != (PyCodeObject *)prev_code) { @@ -13467,14 +14088,18 @@ tracer->prev_state.instr_frame = frame; tracer->prev_state.instr_oparg = oparg; tracer->prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + // Branch opcodes use the cache for branch history, not + // specialization counters. Don't reset it. + && !IS_CONDITIONAL_JUMP_OPCODE(opcode)) { (&next_instr[1])->counter = trigger_backoff_counter(); } - uint8_t record_func_index = _PyOpcode_RecordFunctionIndices[opcode]; - if (record_func_index) { - _Py_RecordFuncPtr doesnt_escape = _PyOpcode_RecordFunctions[record_func_index]; - doesnt_escape(frame, stack_pointer, oparg, &tracer->prev_state.recorded_value); + const _PyOpcodeRecordEntry *record_entry = &_PyOpcode_RecordEntries[opcode]; + for (int i = 0; i < record_entry->count; i++) { + _Py_RecordFuncPtr doesnt_escape = _PyOpcode_RecordFunctions[record_entry->indices[i]]; + doesnt_escape(frame, stack_pointer, oparg, &tracer->prev_state.recorded_values[i]); } + tracer->prev_state.recorded_count = record_entry->count; DISPATCH_GOTO_NON_TRACING(); #else (void)prev_instr; @@ -13864,41 +14489,50 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(YIELD_VALUE); - _PyStackRef retval; + opcode = YIELD_VALUE; _PyStackRef value; - retval = stack_pointer[-1]; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - frame->instr_ptr++; - PyGenObject* gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - _PyStackRef temp = retval; - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame* gen_frame = frame; - frame = tstate->current_frame = frame->previous; - CI_SET_ADAPTIVE_INTERPRETER_ENABLED_STATE - gen_frame->previous = NULL; - ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; - FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert( - frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - value = PyStackRef_MakeHeapSafe(temp); - LLTRACE_RESUME_FRAME(); + _PyStackRef retval; + // _MAKE_HEAP_SAFE + { + value = stack_pointer[-1]; + value = PyStackRef_MakeHeapSafe(value); + } + // _YIELD_VALUE + { + retval = value; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + _PyStackRef temp = retval; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + DTRACE_FUNCTION_RETURN(); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; + FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE && defined(Py_DEBUG) + if (!PyStackRef_IsNone(frame->f_executable)) { + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; + assert(opcode == SEND || opcode == FOR_ITER); + } + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + value = temp; + LLTRACE_RESUME_FRAME(); + } stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -14024,6 +14658,13 @@ JUMP_TO_LABEL(error); } LABEL(exit_unwind) + { + assert(_PyErr_Occurred(tstate)); + DTRACE_FUNCTION_RETURN(); + JUMP_TO_LABEL(exit_unwind_notrace); + } + + LABEL(exit_unwind_notrace) { assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); @@ -14069,12 +14710,15 @@ JUMP_TO_LABEL(error); assert(!_PyErr_Occurred(tstate)); #endif stack_pointer = _PyFrame_GetStackPointer(frame); - #if Py_TAIL_CALL_INTERP + #if _Py_TAIL_CALL_INTERP int opcode; #endif DISPATCH(); } + #if _Py_TAIL_CALL_INTERP && !defined(_Py_TIER2) + Py_GCC_ATTRIBUTE((unused)) + #endif LABEL(stop_tracing) { #if _Py_TIER2 diff --git a/cinderx/Interpreter/3.15/borrowed-ceval.c.template b/cinderx/Interpreter/3.15/borrowed-ceval.c.template index 3a5add430..f2c34954b 100644 --- a/cinderx/Interpreter/3.15/borrowed-ceval.c.template +++ b/cinderx/Interpreter/3.15/borrowed-ceval.c.template @@ -19,7 +19,6 @@ #include "internal/pycore_call.h" #include "internal/pycore_floatobject.h" #include "internal/pycore_tuple.h" -#include "Python/ceval_macros.h" #define _PyCoro_GetAwaitableIter JitCoro_GetAwaitableIter #define _PyEval_GetAwaitable Ci_PyEval_GetAwaitable @@ -31,8 +30,3 @@ #undef _PyCoro_GetAwaitableIter #undef _PyEval_GetAwaitable #undef _PyEval_GetANext - -#if Py_TAIL_CALL_INTERP -#include "cinderx/Interpreter/cinderx_opcode_targets.h" -#include "cinderx/Interpreter/3.15/Includes/generated_cases.c.h" -#endif diff --git a/cinderx/Interpreter/3.15/ceval.h b/cinderx/Interpreter/3.15/ceval.h index 995d27181..02533d49b 100644 --- a/cinderx/Interpreter/3.15/ceval.h +++ b/cinderx/Interpreter/3.15/ceval.h @@ -22,7 +22,6 @@ #include "internal/pycore_call.h" #include "internal/pycore_floatobject.h" #include "internal/pycore_tuple.h" -#include "Python/ceval_macros.h" #define _PyCoro_GetAwaitableIter JitCoro_GetAwaitableIter #define _PyEval_GetAwaitable Ci_PyEval_GetAwaitable @@ -93,8 +92,3 @@ _PyEval_GetAwaitable(PyObject *iterable, int oparg) #undef _PyCoro_GetAwaitableIter #undef _PyEval_GetAwaitable #undef _PyEval_GetANext - -#if Py_TAIL_CALL_INTERP -#include "cinderx/Interpreter/cinderx_opcode_targets.h" -#include "cinderx/Interpreter/3.15/Includes/generated_cases.c.h" -#endif diff --git a/cinderx/Interpreter/3.15/cinder-bytecodes.c b/cinderx/Interpreter/3.15/cinder-bytecodes.c index 4d39f5410..241bc0a33 100644 --- a/cinderx/Interpreter/3.15/cinder-bytecodes.c +++ b/cinderx/Interpreter/3.15/cinder-bytecodes.c @@ -40,6 +40,9 @@ #include "pydtrace.h" #include "setobject.h" + +#include "cinderx/module_c_state.h" + #define USE_COMPUTED_GOTOS 0 #include "Python/ceval_macros.h" @@ -138,6 +141,13 @@ static PyObject* dummy_func( switch (opcode) { // BEGIN BYTECODES // + override inst(LOAD_COMMON_CONSTANT, ( -- value)) { + // Use our own copy of common constants to avoid depending on the + // offset of interp->common_consts within PyInterpreterState. + assert(oparg < NUM_COMMON_CONSTANTS); + value = PyStackRef_FromPyObjectNew(Ci_common_consts[oparg]); + } + override op(_PUSH_FRAME, (new_frame--)) { // Write it out explicitly because it's subtly different. // Eventually this should be the only occurrence of this code. @@ -1100,7 +1110,7 @@ static PyObject* dummy_func( assert(!_PyErr_Occurred(tstate)); #endif RELOAD_STACK(); -#if Py_TAIL_CALL_INTERP +#if _Py_TAIL_CALL_INTERP int opcode; #endif DISPATCH(); diff --git a/cinderx/Interpreter/3.15/cinder_opcode_ids.h b/cinderx/Interpreter/3.15/cinder_opcode_ids.h index 4ecd9f9dc..726665595 100644 --- a/cinderx/Interpreter/3.15/cinder_opcode_ids.h +++ b/cinderx/Interpreter/3.15/cinder_opcode_ids.h @@ -11,44 +11,44 @@ #define INVOKE_METHOD (1 | EXTENDED_OPCODE_FLAG) #define LOAD_FIELD (2 | EXTENDED_OPCODE_FLAG) -#define LOAD_OBJ_FIELD (4 | EXTENDED_OPCODE_FLAG) -#define LOAD_PRIMITIVE_FIELD (5 | EXTENDED_OPCODE_FLAG) -#define STORE_FIELD (6 | EXTENDED_OPCODE_FLAG) -#define STORE_OBJ_FIELD (7 | EXTENDED_OPCODE_FLAG) -#define STORE_PRIMITIVE_FIELD (8 | EXTENDED_OPCODE_FLAG) -#define BUILD_CHECKED_LIST (9 | EXTENDED_OPCODE_FLAG) -#define BUILD_CHECKED_LIST_CACHED (10 | EXTENDED_OPCODE_FLAG) -#define LOAD_TYPE (11 | EXTENDED_OPCODE_FLAG) -#define CAST (12 | EXTENDED_OPCODE_FLAG) -#define CAST_CACHED (13 | EXTENDED_OPCODE_FLAG) -#define LOAD_LOCAL (14 | EXTENDED_OPCODE_FLAG) -#define STORE_LOCAL (15 | EXTENDED_OPCODE_FLAG) -#define STORE_LOCAL_CACHED (16 | EXTENDED_OPCODE_FLAG) -#define PRIMITIVE_BOX (17 | EXTENDED_OPCODE_FLAG) -#define POP_JUMP_IF_ZERO (100 | EXTENDED_OPCODE_FLAG) -#define POP_JUMP_IF_NONZERO (103 | EXTENDED_OPCODE_FLAG) -#define PRIMITIVE_UNBOX (18 | EXTENDED_OPCODE_FLAG) -#define PRIMITIVE_BINARY_OP (19 | EXTENDED_OPCODE_FLAG) -#define PRIMITIVE_UNARY_OP (20 | EXTENDED_OPCODE_FLAG) -#define PRIMITIVE_COMPARE_OP (21 | EXTENDED_OPCODE_FLAG) -#define LOAD_ITERABLE_ARG (22 | EXTENDED_OPCODE_FLAG) -#define LOAD_MAPPING_ARG (23 | EXTENDED_OPCODE_FLAG) -#define INVOKE_FUNCTION (24 | EXTENDED_OPCODE_FLAG) -#define INVOKE_FUNCTION_CACHED (25 | EXTENDED_OPCODE_FLAG) -#define INVOKE_INDIRECT_CACHED (26 | EXTENDED_OPCODE_FLAG) -#define FAST_LEN (27 | EXTENDED_OPCODE_FLAG) -#define CONVERT_PRIMITIVE (28 | EXTENDED_OPCODE_FLAG) -#define INVOKE_NATIVE (29 | EXTENDED_OPCODE_FLAG) -#define LOAD_CLASS (30 | EXTENDED_OPCODE_FLAG) -#define BUILD_CHECKED_MAP (31 | EXTENDED_OPCODE_FLAG) -#define BUILD_CHECKED_MAP_CACHED (32 | EXTENDED_OPCODE_FLAG) -#define SEQUENCE_GET (33 | EXTENDED_OPCODE_FLAG) -#define SEQUENCE_SET (34 | EXTENDED_OPCODE_FLAG) -#define LIST_DEL (35 | EXTENDED_OPCODE_FLAG) -#define REFINE_TYPE (36 | EXTENDED_OPCODE_FLAG) -#define PRIMITIVE_LOAD_CONST (37 | EXTENDED_OPCODE_FLAG) -#define RETURN_PRIMITIVE (40 | EXTENDED_OPCODE_FLAG) -#define TP_ALLOC (41 | EXTENDED_OPCODE_FLAG) -#define TP_ALLOC_CACHED (42 | EXTENDED_OPCODE_FLAG) -#define LOAD_METHOD_STATIC (43 | EXTENDED_OPCODE_FLAG) -#define LOAD_METHOD_STATIC_CACHED (45 | EXTENDED_OPCODE_FLAG) +#define LOAD_OBJ_FIELD (5 | EXTENDED_OPCODE_FLAG) +#define LOAD_PRIMITIVE_FIELD (6 | EXTENDED_OPCODE_FLAG) +#define STORE_FIELD (7 | EXTENDED_OPCODE_FLAG) +#define STORE_OBJ_FIELD (8 | EXTENDED_OPCODE_FLAG) +#define STORE_PRIMITIVE_FIELD (9 | EXTENDED_OPCODE_FLAG) +#define BUILD_CHECKED_LIST (10 | EXTENDED_OPCODE_FLAG) +#define BUILD_CHECKED_LIST_CACHED (11 | EXTENDED_OPCODE_FLAG) +#define LOAD_TYPE (12 | EXTENDED_OPCODE_FLAG) +#define CAST (13 | EXTENDED_OPCODE_FLAG) +#define CAST_CACHED (14 | EXTENDED_OPCODE_FLAG) +#define LOAD_LOCAL (15 | EXTENDED_OPCODE_FLAG) +#define STORE_LOCAL (16 | EXTENDED_OPCODE_FLAG) +#define STORE_LOCAL_CACHED (17 | EXTENDED_OPCODE_FLAG) +#define PRIMITIVE_BOX (18 | EXTENDED_OPCODE_FLAG) +#define POP_JUMP_IF_ZERO (99 | EXTENDED_OPCODE_FLAG) +#define POP_JUMP_IF_NONZERO (102 | EXTENDED_OPCODE_FLAG) +#define PRIMITIVE_UNBOX (19 | EXTENDED_OPCODE_FLAG) +#define PRIMITIVE_BINARY_OP (20 | EXTENDED_OPCODE_FLAG) +#define PRIMITIVE_UNARY_OP (21 | EXTENDED_OPCODE_FLAG) +#define PRIMITIVE_COMPARE_OP (22 | EXTENDED_OPCODE_FLAG) +#define LOAD_ITERABLE_ARG (23 | EXTENDED_OPCODE_FLAG) +#define LOAD_MAPPING_ARG (24 | EXTENDED_OPCODE_FLAG) +#define INVOKE_FUNCTION (25 | EXTENDED_OPCODE_FLAG) +#define INVOKE_FUNCTION_CACHED (26 | EXTENDED_OPCODE_FLAG) +#define INVOKE_INDIRECT_CACHED (27 | EXTENDED_OPCODE_FLAG) +#define FAST_LEN (28 | EXTENDED_OPCODE_FLAG) +#define CONVERT_PRIMITIVE (29 | EXTENDED_OPCODE_FLAG) +#define INVOKE_NATIVE (30 | EXTENDED_OPCODE_FLAG) +#define LOAD_CLASS (31 | EXTENDED_OPCODE_FLAG) +#define BUILD_CHECKED_MAP (32 | EXTENDED_OPCODE_FLAG) +#define BUILD_CHECKED_MAP_CACHED (33 | EXTENDED_OPCODE_FLAG) +#define SEQUENCE_GET (34 | EXTENDED_OPCODE_FLAG) +#define SEQUENCE_SET (35 | EXTENDED_OPCODE_FLAG) +#define LIST_DEL (38 | EXTENDED_OPCODE_FLAG) +#define REFINE_TYPE (39 | EXTENDED_OPCODE_FLAG) +#define PRIMITIVE_LOAD_CONST (40 | EXTENDED_OPCODE_FLAG) +#define RETURN_PRIMITIVE (41 | EXTENDED_OPCODE_FLAG) +#define TP_ALLOC (43 | EXTENDED_OPCODE_FLAG) +#define TP_ALLOC_CACHED (44 | EXTENDED_OPCODE_FLAG) +#define LOAD_METHOD_STATIC (45 | EXTENDED_OPCODE_FLAG) +#define LOAD_METHOD_STATIC_CACHED (46 | EXTENDED_OPCODE_FLAG) diff --git a/cinderx/Interpreter/3.15/cinderx_opcode_targets.h b/cinderx/Interpreter/3.15/cinderx_opcode_targets.h index e4c767d3f..01ff598d1 100644 --- a/cinderx/Interpreter/3.15/cinderx_opcode_targets.h +++ b/cinderx/Interpreter/3.15/cinderx_opcode_targets.h @@ -16,10 +16,8 @@ static void *opcode_targets_table[256] = { &&TARGET_FORMAT_WITH_SPEC, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, - &&TARGET_GET_ITER, - &&TARGET_RESERVED, &&TARGET_GET_LEN, - &&TARGET_GET_YIELD_FROM_ITER, + &&TARGET_RESERVED, &&TARGET_INTERPRETER_EXIT, &&TARGET_LOAD_BUILD_CLASS, &&TARGET_LOAD_LOCALS, @@ -72,6 +70,7 @@ static void *opcode_targets_table[256] = { &&TARGET_EXTENDED_ARG, &&TARGET_FOR_ITER, &&TARGET_GET_AWAITABLE, + &&TARGET_GET_ITER, &&TARGET_IMPORT_FROM, &&TARGET_IMPORT_NAME, &&TARGET_IS_OP, @@ -126,6 +125,7 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, + &&_unknown_opcode, &&TARGET_EXTENDED_OPCODE, &&_unknown_opcode, &&TARGET_RESUME, @@ -178,6 +178,9 @@ static void *opcode_targets_table[256] = { &&TARGET_FOR_ITER_LIST, &&TARGET_FOR_ITER_RANGE, &&TARGET_FOR_ITER_TUPLE, + &&TARGET_FOR_ITER_VIRTUAL, + &&TARGET_GET_ITER_SELF, + &&TARGET_GET_ITER_VIRTUAL, &&TARGET_JUMP_BACKWARD_JIT, &&TARGET_JUMP_BACKWARD_NO_JIT, &&TARGET_LOAD_ATTR_CLASS, @@ -198,7 +201,10 @@ static void *opcode_targets_table[256] = { &&TARGET_LOAD_SUPER_ATTR_ATTR, &&TARGET_LOAD_SUPER_ATTR_METHOD, &&TARGET_RESUME_CHECK, + &&TARGET_RESUME_CHECK_JIT, + &&TARGET_SEND_ASYNC_GEN, &&TARGET_SEND_GEN, + &&TARGET_SEND_VIRTUAL, &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, @@ -227,12 +233,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, @@ -379,7 +379,7 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&TARGET_TRACE_RECORD, + &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -472,12 +472,12 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -527,6 +527,7 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_error(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); @@ -622,12 +623,14 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_VIRTUAL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_SELF(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_VIRTUAL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); -static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); @@ -719,10 +722,13 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK_JIT(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_ASYNC_GEN(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_VIRTUAL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); @@ -864,12 +870,14 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [FOR_ITER_LIST] = _TAIL_CALL_FOR_ITER_LIST, [FOR_ITER_RANGE] = _TAIL_CALL_FOR_ITER_RANGE, [FOR_ITER_TUPLE] = _TAIL_CALL_FOR_ITER_TUPLE, + [FOR_ITER_VIRTUAL] = _TAIL_CALL_FOR_ITER_VIRTUAL, [GET_AITER] = _TAIL_CALL_GET_AITER, [GET_ANEXT] = _TAIL_CALL_GET_ANEXT, [GET_AWAITABLE] = _TAIL_CALL_GET_AWAITABLE, [GET_ITER] = _TAIL_CALL_GET_ITER, + [GET_ITER_SELF] = _TAIL_CALL_GET_ITER_SELF, + [GET_ITER_VIRTUAL] = _TAIL_CALL_GET_ITER_VIRTUAL, [GET_LEN] = _TAIL_CALL_GET_LEN, - [GET_YIELD_FROM_ITER] = _TAIL_CALL_GET_YIELD_FROM_ITER, [IMPORT_FROM] = _TAIL_CALL_IMPORT_FROM, [IMPORT_NAME] = _TAIL_CALL_IMPORT_NAME, [INSTRUMENTED_CALL] = _TAIL_CALL_INSTRUMENTED_CALL, @@ -961,10 +969,13 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [RESERVED] = _TAIL_CALL_RESERVED, [RESUME] = _TAIL_CALL_RESUME, [RESUME_CHECK] = _TAIL_CALL_RESUME_CHECK, + [RESUME_CHECK_JIT] = _TAIL_CALL_RESUME_CHECK_JIT, [RETURN_GENERATOR] = _TAIL_CALL_RETURN_GENERATOR, [RETURN_VALUE] = _TAIL_CALL_RETURN_VALUE, [SEND] = _TAIL_CALL_SEND, + [SEND_ASYNC_GEN] = _TAIL_CALL_SEND_ASYNC_GEN, [SEND_GEN] = _TAIL_CALL_SEND_GEN, + [SEND_VIRTUAL] = _TAIL_CALL_SEND_VIRTUAL, [SETUP_ANNOTATIONS] = _TAIL_CALL_SETUP_ANNOTATIONS, [SET_ADD] = _TAIL_CALL_SET_ADD, [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_SET_FUNCTION_ATTRIBUTE, @@ -1002,18 +1013,13 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, + [120] = _TAIL_CALL_UNKNOWN_OPCODE, [121] = _TAIL_CALL_UNKNOWN_OPCODE, [122] = _TAIL_CALL_UNKNOWN_OPCODE, [123] = _TAIL_CALL_UNKNOWN_OPCODE, [124] = _TAIL_CALL_UNKNOWN_OPCODE, [125] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [213] = _TAIL_CALL_UNKNOWN_OPCODE, - [214] = _TAIL_CALL_UNKNOWN_OPCODE, - [215] = _TAIL_CALL_UNKNOWN_OPCODE, - [216] = _TAIL_CALL_UNKNOWN_OPCODE, - [217] = _TAIL_CALL_UNKNOWN_OPCODE, - [218] = _TAIL_CALL_UNKNOWN_OPCODE, [219] = _TAIL_CALL_UNKNOWN_OPCODE, [220] = _TAIL_CALL_UNKNOWN_OPCODE, [221] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1122,12 +1128,14 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [FOR_ITER_LIST] = _TAIL_CALL_TRACE_RECORD, [FOR_ITER_RANGE] = _TAIL_CALL_TRACE_RECORD, [FOR_ITER_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_VIRTUAL] = _TAIL_CALL_TRACE_RECORD, [GET_AITER] = _TAIL_CALL_TRACE_RECORD, [GET_ANEXT] = _TAIL_CALL_TRACE_RECORD, [GET_AWAITABLE] = _TAIL_CALL_TRACE_RECORD, [GET_ITER] = _TAIL_CALL_TRACE_RECORD, + [GET_ITER_SELF] = _TAIL_CALL_TRACE_RECORD, + [GET_ITER_VIRTUAL] = _TAIL_CALL_TRACE_RECORD, [GET_LEN] = _TAIL_CALL_TRACE_RECORD, - [GET_YIELD_FROM_ITER] = _TAIL_CALL_TRACE_RECORD, [IMPORT_FROM] = _TAIL_CALL_TRACE_RECORD, [IMPORT_NAME] = _TAIL_CALL_TRACE_RECORD, [INSTRUMENTED_CALL] = _TAIL_CALL_TRACE_RECORD, @@ -1219,10 +1227,13 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [RESERVED] = _TAIL_CALL_TRACE_RECORD, [RESUME] = _TAIL_CALL_TRACE_RECORD, [RESUME_CHECK] = _TAIL_CALL_TRACE_RECORD, + [RESUME_CHECK_JIT] = _TAIL_CALL_TRACE_RECORD, [RETURN_GENERATOR] = _TAIL_CALL_TRACE_RECORD, [RETURN_VALUE] = _TAIL_CALL_TRACE_RECORD, [SEND] = _TAIL_CALL_TRACE_RECORD, + [SEND_ASYNC_GEN] = _TAIL_CALL_TRACE_RECORD, [SEND_GEN] = _TAIL_CALL_TRACE_RECORD, + [SEND_VIRTUAL] = _TAIL_CALL_TRACE_RECORD, [SETUP_ANNOTATIONS] = _TAIL_CALL_TRACE_RECORD, [SET_ADD] = _TAIL_CALL_TRACE_RECORD, [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_TRACE_RECORD, @@ -1260,18 +1271,13 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACE_RECORD, [WITH_EXCEPT_START] = _TAIL_CALL_TRACE_RECORD, [YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, + [120] = _TAIL_CALL_UNKNOWN_OPCODE, [121] = _TAIL_CALL_UNKNOWN_OPCODE, [122] = _TAIL_CALL_UNKNOWN_OPCODE, [123] = _TAIL_CALL_UNKNOWN_OPCODE, [124] = _TAIL_CALL_UNKNOWN_OPCODE, [125] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [213] = _TAIL_CALL_UNKNOWN_OPCODE, - [214] = _TAIL_CALL_UNKNOWN_OPCODE, - [215] = _TAIL_CALL_UNKNOWN_OPCODE, - [216] = _TAIL_CALL_UNKNOWN_OPCODE, - [217] = _TAIL_CALL_UNKNOWN_OPCODE, - [218] = _TAIL_CALL_UNKNOWN_OPCODE, [219] = _TAIL_CALL_UNKNOWN_OPCODE, [220] = _TAIL_CALL_UNKNOWN_OPCODE, [221] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/cinderx/Interpreter/3.15/interpreter.c b/cinderx/Interpreter/3.15/interpreter.c index e5383c59f..ee5bdda67 100644 --- a/cinderx/Interpreter/3.15/interpreter.c +++ b/cinderx/Interpreter/3.15/interpreter.c @@ -22,6 +22,10 @@ #include "internal/pycore_stackref.h" #include "internal/pycore_interpframe.h" +#ifndef _PyThreadState_UpdateLastProfiledFrame +#define _PyThreadState_UpdateLastProfiledFrame(tstate, frame, previous) +#endif + #include "cinderx/StaticPython/classloader.h" #include "cinderx/StaticPython/checked_dict.h" #include "cinderx/StaticPython/checked_list.h" @@ -29,13 +33,40 @@ #include "cinderx/Jit/generators_rt.h" -#ifdef ENABLE_INTERPRETER_LOOP +#undef EXTRA_CASES + +#define EXTRA_CASES \ + case 120: \ + case 122: \ + case 123: \ + case 124: \ + case 125: \ + case 127: \ + case 214: \ + case 215: \ + case 216: \ + case 217: \ + case 218: \ + case 219: \ + case 220: \ + case 221: \ + case 222: \ + case 223: \ + case 224: \ + case 225: \ + case 226: \ + case 227: \ + case 228: \ + case 229: \ + case 230: \ + case 231: \ + case 232: \ + ; -bool Ci_DelayAdaptiveCode = false; -uint64_t Ci_AdaptiveThreshold = 80; +#ifdef ENABLE_INTERPRETER_LOOP bool is_adaptive_enabled(CodeExtra *extra) { - return !Ci_DelayAdaptiveCode || Ci_code_extra_get_calls(extra) > Ci_AdaptiveThreshold; + return !Ci_GetDelayAdaptiveCode() || Ci_code_extra_get_calls(extra) > Ci_GetAdaptiveThreshold(); } #endif @@ -379,6 +410,7 @@ Ci_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag); #include "cinderx/Interpreter/3.15/ceval.h" #include "Python/ceval.h" +#include "cinderx/Interpreter/3.15/Includes/ceval_macros.h" #endif @@ -478,14 +510,17 @@ Py_ssize_t load_method_static_cached_oparg_slot(int oparg) { (tstate->interp->eval_frame != NULL && \ tstate->interp->eval_frame != Ci_EvalFrame) - - #ifdef ENABLE_INTERPRETER_LOOP +#if _Py_TAIL_CALL_INTERP +#include "cinderx/Interpreter/cinderx_opcode_targets.h" +#include "cinderx/Interpreter/3.15/Includes/generated_cases.c.h" +#endif + PyObject* _Py_HOT_FUNCTION Ci_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { -#if USE_COMPUTED_GOTOS && !Py_TAIL_CALL_INTERP +#if USE_COMPUTED_GOTOS && !_Py_TAIL_CALL_INTERP /* Import the static jump table */ #include "cinderx/Interpreter/cinderx_opcode_targets.h" void **opcode_targets = opcode_targets_table; @@ -494,7 +529,7 @@ void **opcode_targets = opcode_targets_table; #ifdef Py_STATS int lastopcode = 0; #endif -#if !Py_TAIL_CALL_INTERP +#if !_Py_TAIL_CALL_INTERP uint8_t opcode; /* Current opcode */ int oparg; /* Current opcode argument, if any */ assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL); @@ -572,11 +607,11 @@ void **opcode_targets = opcode_targets_table; next_instr = frame->instr_ptr; monitor_throw(tstate, frame, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); -#if Py_TAIL_CALL_INTERP +#if _Py_TAIL_CALL_INTERP # if Py_STATS - return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, 0, lastopcode, adaptive_enabled); + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0, lastopcode, adaptive_enabled); # else - return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, 0, adaptive_enabled); + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0, adaptive_enabled); # endif #else goto error; @@ -588,11 +623,11 @@ void **opcode_targets = opcode_targets_table; _PyExecutorObject *current_executor = NULL; const _PyUOpInstruction *next_uop = NULL; #endif -#if Py_TAIL_CALL_INTERP +#if _Py_TAIL_CALL_INTERP # if Py_STATS - return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, 0, lastopcode, adaptive_enabled); + return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_handler_table, 0, lastopcode, adaptive_enabled); # else - return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, 0, adaptive_enabled); + return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_handler_table, 0, adaptive_enabled); # endif #else goto start_frame; diff --git a/cinderx/Interpreter/3.16/.clang-format b/cinderx/Interpreter/3.16/.clang-format new file mode 100644 index 000000000..e871ed18b --- /dev/null +++ b/cinderx/Interpreter/3.16/.clang-format @@ -0,0 +1,3 @@ +--- +DisableFormat: true +SortIncludes: false diff --git a/cinderx/Interpreter/3.16/Includes/Python/README.txt b/cinderx/Interpreter/3.16/Includes/Python/README.txt new file mode 100644 index 000000000..99fcb7792 --- /dev/null +++ b/cinderx/Interpreter/3.16/Includes/Python/README.txt @@ -0,0 +1 @@ +# These are vendored in CPython internal header files. diff --git a/cinderx/Interpreter/3.16/Includes/Python/ceval.h b/cinderx/Interpreter/3.16/Includes/Python/ceval.h new file mode 100644 index 000000000..0437ab85c --- /dev/null +++ b/cinderx/Interpreter/3.16/Includes/Python/ceval.h @@ -0,0 +1,626 @@ +#define _PY_INTERPRETER + +#include "Python.h" +#include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_audit.h" // _PySys_Audit() +#include "pycore_backoff.h" +#include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_cell.h" // PyCell_GetRef() +#include "pycore_ceval.h" // SPECIAL___ENTER__ +#include "pycore_code.h" +#include "pycore_dict.h" +#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS +#include "pycore_floatobject.h" // _PyFloat_ExactDealloc() +#include "pycore_frame.h" +#include "pycore_function.h" +#include "pycore_genobject.h" // _PyCoro_GetAwaitableIter() +#include "pycore_import.h" // _PyImport_IsDefaultImportFunc() +#include "pycore_instruments.h" +#include "pycore_interpframe.h" // _PyFrame_SetStackPointer() +#include "pycore_interpolation.h" // _PyInterpolation_Build() +#include "pycore_intrinsics.h" +#include "pycore_jit.h" +#include "pycore_lazyimportobject.h" +#include "pycore_list.h" // _PyList_GetItemRef() +#include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_moduleobject.h" // PyModuleObject +#include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_opcode_metadata.h" // EXTRA_CASES +#include "pycore_opcode_utils.h" // MAKE_FUNCTION_* +#include "pycore_optimizer.h" // _PyUOpExecutor_Type +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() +#include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_range.h" // _PyRangeIterObject +#include "pycore_setobject.h" // _PySet_Update() +#include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs +#include "pycore_sysmodule.h" // _PySys_GetOptionalAttrString() +#include "pycore_template.h" // _PyTemplate_Build() +#include "pycore_traceback.h" // _PyTraceBack_FromFrame +#include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_uop_ids.h" // Uops + +#include "dictobject.h" +#include "frameobject.h" // _PyInterpreterFrame_GetLine +#include "opcode.h" +#include "pydtrace.h" +#include "setobject.h" +#include "pycore_stackref.h" + +#include // bool + +#if !defined(Py_BUILD_CORE) +# error "ceval.c must be build with Py_BUILD_CORE define for best performance" +#endif + +#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) +// GH-89279: The MSVC compiler does not inline these static inline functions +// in PGO build in _PyEval_EvalFrameDefault(), because this function is over +// the limit of PGO, and that limit cannot be configured. +// Define them as macros to make sure that they are always inlined by the +// preprocessor. + +#undef Py_IS_TYPE +#define Py_IS_TYPE(ob, type) \ + (_PyObject_CAST(ob)->ob_type == (type)) + +#undef Py_XDECREF +#define Py_XDECREF(arg) \ + do { \ + PyObject *xop = _PyObject_CAST(arg); \ + if (xop != NULL) { \ + Py_DECREF(xop); \ + } \ + } while (0) + +#ifndef Py_GIL_DISABLED + +#undef Py_DECREF +#define Py_DECREF(arg) \ + do { \ + PyObject *op = _PyObject_CAST(arg); \ + if (_Py_IsImmortal(op)) { \ + _Py_DECREF_IMMORTAL_STAT_INC(); \ + break; \ + } \ + _Py_DECREF_STAT_INC(); \ + if (--op->ob_refcnt == 0) { \ + _PyReftracerTrack(op, PyRefTracer_DESTROY); \ + destructor dealloc = Py_TYPE(op)->tp_dealloc; \ + (*dealloc)(op); \ + } \ + } while (0) + +#undef _Py_DECREF_SPECIALIZED +#define _Py_DECREF_SPECIALIZED(arg, dealloc) \ + do { \ + PyObject *op = _PyObject_CAST(arg); \ + if (_Py_IsImmortal(op)) { \ + _Py_DECREF_IMMORTAL_STAT_INC(); \ + break; \ + } \ + _Py_DECREF_STAT_INC(); \ + if (--op->ob_refcnt == 0) { \ + _PyReftracerTrack(op, PyRefTracer_DESTROY); \ + destructor d = (destructor)(dealloc); \ + d(op); \ + } \ + } while (0) + +#else // Py_GIL_DISABLED + +#undef Py_DECREF +#define Py_DECREF(arg) \ + do { \ + PyObject *op = _PyObject_CAST(arg); \ + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); \ + if (local == _Py_IMMORTAL_REFCNT_LOCAL) { \ + _Py_DECREF_IMMORTAL_STAT_INC(); \ + break; \ + } \ + _Py_DECREF_STAT_INC(); \ + if (_Py_IsOwnedByCurrentThread(op)) { \ + local--; \ + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); \ + if (local == 0) { \ + _Py_MergeZeroLocalRefcount(op); \ + } \ + } \ + else { \ + _Py_DecRefShared(op); \ + } \ + } while (0) + +#undef _Py_DECREF_SPECIALIZED +#define _Py_DECREF_SPECIALIZED(arg, dealloc) Py_DECREF(arg) + +#endif +#endif + +static void +check_invalid_reentrancy(void) +{ +#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED) + // In the free-threaded build, the interpreter must not be re-entered if + // the world-is-stopped. If so, that's a bug somewhere (quite likely in + // the painfully complex typeobject code). + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(!interp->stoptheworld.world_stopped); +#endif +} + + +#ifdef Py_DEBUG +static void +dump_item(_PyStackRef item) +{ + if (PyStackRef_IsNull(item)) { + printf(""); + return; + } + if (PyStackRef_IsMalformed(item)) { + printf(""); + return; + } + if (PyStackRef_IsTaggedInt(item)) { + printf("%" PRId64, (int64_t)PyStackRef_UntagInt(item)); + return; + } + PyObject *obj = PyStackRef_AsPyObjectBorrow(item); + if (obj == NULL) { + printf(""); + return; + } + // Don't call __repr__(), it might recurse into the interpreter. + printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)obj); +} + +static void +dump_stack(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) +{ + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef *locals_base = _PyFrame_GetLocalsArray(frame); + _PyStackRef *stack_base = _PyFrame_Stackbase(frame); + PyObject *exc = PyErr_GetRaisedException(); + printf(" locals=["); + for (_PyStackRef *ptr = locals_base; ptr < stack_base; ptr++) { + if (ptr != locals_base) { + printf(", "); + } + dump_item(*ptr); + } + printf("]\n"); + if (stack_pointer < stack_base) { + printf(" stack=%d\n", (int)(stack_pointer-stack_base)); + } + else { + printf(" stack=["); + for (_PyStackRef *ptr = stack_base; ptr < stack_pointer; ptr++) { + if (ptr != stack_base) { + printf(", "); + } + dump_item(*ptr); + } + printf("]\n"); + } + fflush(stdout); + PyErr_SetRaisedException(exc); + _PyFrame_GetStackPointer(frame); +} + +#if defined(_Py_TIER2) && !defined(_Py_JIT) && defined(Py_DEBUG) +static void +dump_cache_item(_PyStackRef cache, int position, int depth) +{ + if (position < depth) { + dump_item(cache); + } + else { + printf("---"); + } +} +#endif + +static void +lltrace_instruction(_PyInterpreterFrame *frame, + _PyStackRef *stack_pointer, + _Py_CODEUNIT *next_instr, + int opcode, + int oparg) +{ + int offset = 0; + if (frame->owner < FRAME_OWNED_BY_INTERPRETER) { + dump_stack(frame, stack_pointer); + offset = (int)(next_instr - _PyFrame_GetBytecode(frame)); + } + const char *opname = _PyOpcode_OpName[opcode]; + assert(opname != NULL); + if (OPCODE_HAS_ARG((int)_PyOpcode_Deopt[opcode])) { + printf("%d: %s %d\n", offset * 2, opname, oparg); + } + else { + printf("%d: %s\n", offset * 2, opname); + } + fflush(stdout); +} + +static void +lltrace_resume_frame(_PyInterpreterFrame *frame) +{ + PyObject *fobj = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + if (!PyStackRef_CodeCheck(frame->f_executable) || + fobj == NULL || + !PyFunction_Check(fobj) + ) { + printf("\nResuming frame.\n"); + return; + } + PyFunctionObject *f = (PyFunctionObject *)fobj; + PyObject *exc = PyErr_GetRaisedException(); + PyObject *name = f->func_qualname; + if (name == NULL) { + name = f->func_name; + } + printf("\nResuming frame"); + if (name) { + printf(" for "); + if (PyObject_Print(name, stdout, 0) < 0) { + PyErr_Clear(); + } + } + if (f->func_module) { + printf(" in module "); + if (PyObject_Print(f->func_module, stdout, 0) < 0) { + PyErr_Clear(); + } + } + printf("\n"); + fflush(stdout); + PyErr_SetRaisedException(exc); +} + +static int +maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, PyObject *globals) +{ + if (globals == NULL) { + return 0; + } + if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { + return 0; + } + int r = PyDict_Contains(globals, &_Py_ID(__lltrace__)); + if (r < 0) { + PyErr_Clear(); + return 0; + } + int lltrace = r * 5; // Levels 1-4 only trace uops + if (!lltrace) { + // Can also be controlled by environment variable + char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); + if (python_lltrace != NULL && *python_lltrace >= '0') { + lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that + } + } + if (lltrace >= 5) { + lltrace_resume_frame(frame); + } + return lltrace; +} + +#endif + +static void monitor_reraise(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr); +static int monitor_stop_iteration(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, + PyObject *value); +static void monitor_unwind(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr); +static int monitor_handled(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, PyObject *exc); +static void monitor_throw(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr); + +static int get_exception_handler(PyCodeObject *, int, int*, int*, int*); + +#ifdef HAVE_ERRNO_H +#include +#endif + +typedef struct { + _PyInterpreterFrame frame; + _PyStackRef stack[1]; +} _PyEntryFrame; + +static int +do_monitor_exc(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, int event) +{ + assert(event < _PY_MONITORING_UNGROUPED_EVENTS); + if (_PyFrame_GetCode(frame)->co_flags & CO_NO_MONITORING_EVENTS) { + return 0; + } + PyObject *exc = PyErr_GetRaisedException(); + assert(exc != NULL); + int err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc); + if (err == 0) { + PyErr_SetRaisedException(exc); + } + else { + assert(PyErr_Occurred()); + Py_DECREF(exc); + } + return err; +} + +static inline bool +no_tools_for_global_event(PyThreadState *tstate, int event) +{ + return tstate->interp->monitors.tools[event] == 0; +} + +static inline bool +no_tools_for_local_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event) +{ + assert(event < _PY_MONITORING_UNGROUPED_EVENTS); + _PyCoMonitoringData *data = _PyFrame_GetCode(frame)->_co_monitoring; + if (data) { + return data->active_monitors.tools[event] == 0; + } + else { + return no_tools_for_global_event(tstate, event); + } +} + +static int +monitor_handled(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, PyObject *exc) +{ + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) { + return 0; + } + return _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc); +} + +static void +monitor_throw(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) +{ + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_PY_THROW)) { + return; + } + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW); +} + +static void +monitor_reraise(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) +{ + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_RERAISE)) { + return; + } + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RERAISE); +} + +static int +monitor_stop_iteration(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr, PyObject *value) +{ + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) { + return 0; + } + assert(!PyErr_Occurred()); + PyErr_SetObject(PyExc_StopIteration, value); + int res = do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION); + if (res < 0) { + return res; + } + PyErr_SetRaisedException(NULL); + return 0; +} + +static void +monitor_unwind(PyThreadState *tstate, + _PyInterpreterFrame *frame, + _Py_CODEUNIT *instr) +{ + if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_PY_UNWIND)) { + return; + } + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_UNWIND); +} + +static inline unsigned char * +scan_back_to_entry_start(unsigned char *p) { + for (; (p[0]&128) == 0; p--); + return p; +} + +static inline unsigned char * +skip_to_next_entry(unsigned char *p, unsigned char *end) { + while (p < end && ((p[0] & 128) == 0)) { + p++; + } + return p; +} + + +#define MAX_LINEAR_SEARCH 40 + +static Py_NO_INLINE int +get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, int *lasti) +{ + unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code->co_exceptiontable); + unsigned char *end = start + PyBytes_GET_SIZE(code->co_exceptiontable); + /* Invariants: + * start_table == end_table OR + * start_table points to a legal entry and end_table points + * beyond the table or to a legal entry that is after index. + */ + if (end - start > MAX_LINEAR_SEARCH) { + int offset; + parse_varint(start, &offset); + if (offset > index) { + return 0; + } + do { + unsigned char * mid = start + ((end-start)>>1); + mid = scan_back_to_entry_start(mid); + parse_varint(mid, &offset); + if (offset > index) { + end = mid; + } + else { + start = mid; + } + + } while (end - start > MAX_LINEAR_SEARCH); + } + unsigned char *scan = start; + while (scan < end) { + int start_offset, size; + scan = parse_varint(scan, &start_offset); + if (start_offset > index) { + break; + } + scan = parse_varint(scan, &size); + if (start_offset + size > index) { + scan = parse_varint(scan, handler); + int depth_and_lasti; + parse_varint(scan, &depth_and_lasti); + *level = depth_and_lasti >> 1; + *lasti = depth_and_lasti & 1; + return 1; + } + scan = skip_to_next_entry(scan, end); + } + return 0; +} + + +#ifdef Py_DEBUG +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_assert_within_stack_bounds(frame, stack_pointer, (F), (L)) +#else +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0 +#endif + +/* Logic for the raise statement (too complicated for inlining). + This *consumes* a reference count to each of its arguments. */ +static int +do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause) +{ + PyObject *type = NULL, *value = NULL; + + if (exc == NULL) { + /* Reraise */ + _PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate); + exc = exc_info->exc_value; + if (Py_IsNone(exc) || exc == NULL) { + _PyErr_SetString(tstate, PyExc_RuntimeError, + "No active exception to reraise"); + return 0; + } + Py_INCREF(exc); + assert(PyExceptionInstance_Check(exc)); + _PyErr_SetRaisedException(tstate, exc); + return 1; + } + + /* We support the following forms of raise: + raise + raise + raise */ + + if (PyExceptionClass_Check(exc)) { + type = exc; + value = _PyObject_CallNoArgs(exc); + if (value == NULL) + goto raise_error; + if (!PyExceptionInstance_Check(value)) { + _PyErr_Format(tstate, PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto raise_error; + } + } + else if (PyExceptionInstance_Check(exc)) { + value = exc; + type = PyExceptionInstance_Class(exc); + Py_INCREF(type); + } + else { + /* Not something you can raise. You get an exception + anyway, just not what you specified :-) */ + Py_DECREF(exc); + _PyErr_SetString(tstate, PyExc_TypeError, + "exceptions must derive from BaseException"); + goto raise_error; + } + + assert(type != NULL); + assert(value != NULL); + + if (cause) { + PyObject *fixed_cause; + if (PyExceptionClass_Check(cause)) { + fixed_cause = _PyObject_CallNoArgs(cause); + if (fixed_cause == NULL) + goto raise_error; + if (!PyExceptionInstance_Check(fixed_cause)) { + _PyErr_Format(tstate, PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + cause, Py_TYPE(fixed_cause)); + Py_DECREF(fixed_cause); + goto raise_error; + } + Py_DECREF(cause); + } + else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + } + else if (Py_IsNone(cause)) { + Py_DECREF(cause); + fixed_cause = NULL; + } + else { + _PyErr_SetString(tstate, PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto raise_error; + } + PyException_SetCause(value, fixed_cause); + } + + _PyErr_SetObject(tstate, type, value); + /* _PyErr_SetObject incref's its arguments */ + Py_DECREF(value); + Py_DECREF(type); + return 0; + +raise_error: + Py_XDECREF(value); + Py_XDECREF(type); + Py_XDECREF(cause); + return 0; +} + +/* Disable unused label warnings. They are handy for debugging, even + if computed gotos aren't used. */ + +/* TBD - what about other compilers? */ +#if defined(__GNUC__) || defined(__clang__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wunused-label" +#elif defined(_MSC_VER) /* MS_WINDOWS */ +# pragma warning(push) +# pragma warning(disable:4102) +#endif diff --git a/cinderx/Interpreter/3.16/Includes/Python/ceval_macros.h b/cinderx/Interpreter/3.16/Includes/Python/ceval_macros.h new file mode 100644 index 000000000..36921e355 --- /dev/null +++ b/cinderx/Interpreter/3.16/Includes/Python/ceval_macros.h @@ -0,0 +1,683 @@ +// Macros and other things needed by ceval.c, and bytecodes.c + +/* Computed GOTOs, or + the-optimization-commonly-but-improperly-known-as-"threaded code" + using gcc's labels-as-values extension + (http://gcc.gnu.org/onlinedocs/gcc/Labels-as-Values.html). + + The traditional bytecode evaluation loop uses a "switch" statement, which + decent compilers will optimize as a single indirect branch instruction + combined with a lookup table of jump addresses. However, since the + indirect jump instruction is shared by all opcodes, the CPU will have a + hard time making the right prediction for where to jump next (actually, + it will be always wrong except in the uncommon case of a sequence of + several identical opcodes). + + "Threaded code" in contrast, uses an explicit jump table and an explicit + indirect jump instruction at the end of each opcode. Since the jump + instruction is at a different address for each opcode, the CPU will make a + separate prediction for each of these instructions, which is equivalent to + predicting the second opcode of each opcode pair. These predictions have + a much better chance to turn out valid, especially in small bytecode loops. + + A mispredicted branch on a modern CPU flushes the whole pipeline and + can cost several CPU cycles (depending on the pipeline depth), + and potentially many more instructions (depending on the pipeline width). + A correctly predicted branch, however, is nearly free. + + At the time of this writing, the "threaded code" version is up to 15-20% + faster than the normal "switch" version, depending on the compiler and the + CPU architecture. + + NOTE: care must be taken that the compiler doesn't try to "optimize" the + indirect jumps by sharing them between all opcodes. Such optimizations + can be disabled on gcc by using the -fno-gcse flag (or possibly + -fno-crossjumping). +*/ + +/* Use macros rather than inline functions, to make it as clear as possible + * to the C compiler that the tracing check is a simple test then branch. + * We want to be sure that the compiler knows this before it generates + * the CFG. + */ + +#ifdef WITH_DTRACE +#define OR_DTRACE_LINE | (PyDTrace_LINE_ENABLED() ? 255 : 0) +#else +#define OR_DTRACE_LINE +#endif + +#ifdef HAVE_COMPUTED_GOTOS + #ifndef USE_COMPUTED_GOTOS + #define USE_COMPUTED_GOTOS 1 + #endif +#else + #if defined(USE_COMPUTED_GOTOS) && USE_COMPUTED_GOTOS + #error "Computed gotos are not supported on this compiler." + #endif + #undef USE_COMPUTED_GOTOS + #define USE_COMPUTED_GOTOS 0 +#endif + +#ifdef Py_STATS +#define INSTRUCTION_STATS(op) \ + do { \ + PyStats *s = _PyStats_GET(); \ + OPCODE_EXE_INC(op); \ + if (s) s->opcode_stats[lastopcode].pair_count[op]++; \ + lastopcode = op; \ + } while (0) +#else +#define INSTRUCTION_STATS(op) ((void)0) +#endif + +#ifdef Py_STATS +# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, const void *instruction_funcptr_table, int oparg, int lastopcode +# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg, lastopcode +#else +# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, const void *instruction_funcptr_table, int oparg +# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg +#endif + +#if _Py_TAIL_CALL_INTERP +# if defined(__clang__) || defined(__GNUC__) +# if !_Py__has_attribute(preserve_none) || !_Py__has_attribute(musttail) +# error "This compiler does not have support for efficient tail calling." +# endif +# elif defined(_MSC_VER) && (_MSC_VER < 1950) +# error "You need at least VS 2026 / PlatformToolset v145 for tail calling." +# endif +# if defined(_MSC_VER) && !defined(__clang__) +# define Py_MUSTTAIL [[msvc::musttail]] +# define Py_PRESERVE_NONE_CC __preserve_none +# else +# define Py_MUSTTAIL __attribute__((musttail)) +# define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) +# endif + typedef PyObject *(Py_PRESERVE_NONE_CC *py_tail_call_funcptr)(TAIL_CALL_PARAMS); + +# define DISPATCH_TABLE_VAR instruction_funcptr_table +# define DISPATCH_TABLE instruction_funcptr_handler_table +# define TRACING_DISPATCH_TABLE instruction_funcptr_tracing_table +# define TARGET(op) Py_NO_INLINE PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_##op(TAIL_CALL_PARAMS) + +# define DISPATCH_GOTO() \ + do { \ + Py_MUSTTAIL return (((py_tail_call_funcptr *)instruction_funcptr_table)[opcode])(TAIL_CALL_ARGS); \ + } while (0) +# define DISPATCH_GOTO_NON_TRACING() \ + do { \ + Py_MUSTTAIL return (((py_tail_call_funcptr *)DISPATCH_TABLE)[opcode])(TAIL_CALL_ARGS); \ + } while (0) +# define JUMP_TO_LABEL(name) \ + do { \ + Py_MUSTTAIL return (_TAIL_CALL_##name)(TAIL_CALL_ARGS); \ + } while (0) +# ifdef Py_STATS +# define JUMP_TO_PREDICTED(name) \ + do { \ + Py_MUSTTAIL return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, this_instr, instruction_funcptr_table, oparg, lastopcode); \ + } while (0) +# else +# define JUMP_TO_PREDICTED(name) \ + do { \ + Py_MUSTTAIL return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, this_instr, instruction_funcptr_table, oparg); \ + } while (0) +# endif +# define LABEL(name) TARGET(name) +#elif USE_COMPUTED_GOTOS +# define DISPATCH_TABLE_VAR opcode_targets +# define DISPATCH_TABLE opcode_targets_table +# define TRACING_DISPATCH_TABLE opcode_tracing_targets_table +# define TARGET(op) TARGET_##op: +# define DISPATCH_GOTO() goto *opcode_targets[opcode] +# define DISPATCH_GOTO_NON_TRACING() goto *DISPATCH_TABLE[opcode]; +# define JUMP_TO_LABEL(name) goto name; +# define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; +# define LABEL(name) name: +#else +# define TARGET(op) case op: TARGET_##op: +# define DISPATCH_GOTO() dispatch_code = opcode | tracing_mode ; goto dispatch_opcode +# define DISPATCH_GOTO_NON_TRACING() dispatch_code = opcode; goto dispatch_opcode +# define JUMP_TO_LABEL(name) goto name; +# define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; +# define LABEL(name) name: +#endif + +#if (_Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS) && _Py_TIER2 +# define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) +# define ENTER_TRACING() \ + DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; +# define LEAVE_TRACING() \ + DISPATCH_TABLE_VAR = DISPATCH_TABLE; +#else +# define IS_JIT_TRACING() (tracing_mode != 0) +# define ENTER_TRACING() tracing_mode = 255 +# define LEAVE_TRACING() tracing_mode = 0 +#endif + +#if _Py_TIER2 +#define STOP_TRACING() \ + do { \ + if (IS_JIT_TRACING()) { \ + LEAVE_TRACING(); \ + _PyJit_FinalizeTracing(tstate, 0); \ + } \ + } while (0); +#else +#define STOP_TRACING() ((void)(0)); +#endif + +/* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ +#ifdef Py_DEBUG +#define PRE_DISPATCH_GOTO() if (frame->lltrace >= 5) { \ + lltrace_instruction(frame, stack_pointer, next_instr, opcode, oparg); } +#else +#define PRE_DISPATCH_GOTO() ((void)0) +#endif + +#ifdef Py_DEBUG +#define LLTRACE_RESUME_FRAME() \ +do { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + frame->lltrace = lltrace; \ +} while (0) +#else +#define LLTRACE_RESUME_FRAME() ((void)0) +#endif + +#ifdef Py_GIL_DISABLED +#define QSBR_QUIESCENT_STATE(tstate) _Py_qsbr_quiescent_state(((_PyThreadStateImpl *)tstate)->qsbr) +#else +#define QSBR_QUIESCENT_STATE(tstate) +#endif + + +/* Do interpreter dispatch accounting for tracing and instrumentation */ +#define DISPATCH() \ + { \ + _PyFrame_StackAssertInvalid(frame); \ + NEXTOPARG(); \ + PRE_DISPATCH_GOTO(); \ + DISPATCH_GOTO(); \ + } + +#define DISPATCH_NON_TRACING() \ + { \ + _PyFrame_StackAssertInvalid(frame); \ + NEXTOPARG(); \ + PRE_DISPATCH_GOTO(); \ + DISPATCH_GOTO_NON_TRACING(); \ + } + +#define DISPATCH_SAME_OPARG() \ + { \ + opcode = next_instr->op.code; \ + PRE_DISPATCH_GOTO(); \ + DISPATCH_GOTO_NON_TRACING(); \ + } + +#define DISPATCH_INLINED(NEW_FRAME) \ + do { \ + assert(!IS_PEP523_HOOKED(tstate)); \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + _PyFrame_StackPointerValidate(frame); \ + assert((NEW_FRAME)->previous == frame); \ + frame = tstate->current_frame = (NEW_FRAME); \ + CALL_STAT_INC(inlined_py_calls); \ + JUMP_TO_LABEL(start_frame); \ + } while (0) + +/* Tuple access macros */ + +#ifndef Py_DEBUG +#define GETITEM(v, i) PyTuple_GET_ITEM((v), (i)) +#else +static inline PyObject * +GETITEM(PyObject *v, Py_ssize_t i) { + assert(PyTuple_Check(v)); + assert(i >= 0); + assert(i < PyTuple_GET_SIZE(v)); + return PyTuple_GET_ITEM(v, i); +} +#endif + +/* Code access macros */ + +/* The integer overflow is checked by an assertion below. */ +#define INSTR_OFFSET() ((int)(next_instr - _PyFrame_GetBytecode(frame))) +#define NEXTOPARG() do { \ + _Py_CODEUNIT word = {.cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t*)next_instr)}; \ + opcode = word.op.code; \ + oparg = word.op.arg; \ + } while (0) + +/* JUMPBY makes the generator identify the instruction as a jump. SKIP_OVER is + * for advancing to the next instruction, taking into account cache entries + * and skipped instructions. + */ +#define JUMPBY(x) (next_instr += (x)) +#define SKIP_OVER(x) (next_instr += (x)) + +#define STACK_LEVEL() ((int)(stack_pointer - _PyFrame_Stackbase(frame))) +#define STACK_SIZE() (_PyFrame_GetCode(frame)->co_stacksize) + +#define WITHIN_STACK_BOUNDS() \ + (frame->owner == FRAME_OWNED_BY_INTERPRETER || (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE())) + +#if defined(Py_DEBUG) && !defined(_Py_JIT) +// This allows temporary stack "overflows", provided it's all in the cache at any point of time. +#define ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(F, L) \ + assert(frame->owner == FRAME_OWNED_BY_INTERPRETER || (STACK_LEVEL() >= 0 && (STACK_LEVEL()) <= STACK_SIZE())) +#else +#define ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE ASSERT_WITHIN_STACK_BOUNDS +#endif + +/* Data access macros */ +#define FRAME_CO_CONSTS (_PyFrame_GetCode(frame)->co_consts) +#define FRAME_CO_NAMES (_PyFrame_GetCode(frame)->co_names) + +/* Local variable macros */ + +#define LOCALS_ARRAY (frame->localsplus) +#define GETLOCAL(i) (frame->localsplus[i]) + + +#ifdef Py_STATS +#define UPDATE_MISS_STATS(INSTNAME) \ + do { \ + STAT_INC(opcode, miss); \ + STAT_INC((INSTNAME), miss); \ + /* The counter is always the first cache entry: */ \ + if (ADAPTIVE_COUNTER_TRIGGERS(next_instr->cache)) { \ + STAT_INC((INSTNAME), deopt); \ + } \ + } while (0) +#else +#define UPDATE_MISS_STATS(INSTNAME) ((void)0) +#endif + + +// Try to lock an object in the free threading build, if it's not already +// locked. Use with a DEOPT_IF() to deopt if the object is already locked. +// These are no-ops in the default GIL build. The general pattern is: +// +// DEOPT_IF(!LOCK_OBJECT(op)); +// if (/* condition fails */) { +// UNLOCK_OBJECT(op); +// DEOPT_IF(true); +// } +// ... +// UNLOCK_OBJECT(op); +// +// NOTE: The object must be unlocked on every exit code path and you should +// avoid any potentially escaping calls (like PyStackRef_CLOSE) while the +// object is locked. +#ifdef Py_GIL_DISABLED +# define LOCK_OBJECT(op) PyMutex_LockFast(&(_PyObject_CAST(op))->ob_mutex) +# define UNLOCK_OBJECT(op) PyMutex_Unlock(&(_PyObject_CAST(op))->ob_mutex) +#else +# define LOCK_OBJECT(op) (1) +# define UNLOCK_OBJECT(op) ((void)0) +#endif + +#define GLOBALS() frame->f_globals +#define BUILTINS() frame->f_builtins +#define LOCALS() frame->f_locals +#define CONSTS() _PyFrame_GetCode(frame)->co_consts +#define NAMES() _PyFrame_GetCode(frame)->co_names + +#if defined(WITH_DTRACE) && !defined(Py_BUILD_CORE_MODULE) +static void dtrace_function_entry(_PyInterpreterFrame *); +static void dtrace_function_return(_PyInterpreterFrame *); + +#define DTRACE_FUNCTION_ENTRY() \ + if (PyDTrace_FUNCTION_ENTRY_ENABLED()) { \ + dtrace_function_entry(frame); \ + } + +#define DTRACE_FUNCTION_RETURN() \ + if (PyDTrace_FUNCTION_RETURN_ENABLED()) { \ + dtrace_function_return(frame); \ + } +#else +#define DTRACE_FUNCTION_ENTRY() ((void)0) +#define DTRACE_FUNCTION_RETURN() ((void)0) +#endif + +/* This takes a uint16_t instead of a _Py_BackoffCounter, + * because it is used directly on the cache entry in generated code, + * which is always an integral type. */ +// Force re-specialization when tracing a side exit to get good side exits. +#define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \ + backoff_counter_triggers(forge_backoff_counter((COUNTER))) + +#ifdef Py_GIL_DISABLED +/* Counters are unreachable when thread-local bytecode is disabled, + * so there is no need to update them. */ +#define ADVANCE_ADAPTIVE_COUNTER(COUNTER) \ + do { \ + _Py_BackoffCounter cnt = (COUNTER); \ + if (!backoff_counter_is_unreachable(cnt)) { \ + (COUNTER) = advance_backoff_counter(cnt); \ + } \ + } while (0); + +#define PAUSE_ADAPTIVE_COUNTER(COUNTER) \ + do { \ + _Py_BackoffCounter cnt = (COUNTER); \ + if (!backoff_counter_is_unreachable(cnt)) { \ + (COUNTER) = pause_backoff_counter(cnt); \ + } \ + } while (0); +#else +#define ADVANCE_ADAPTIVE_COUNTER(COUNTER) \ + do { \ + (COUNTER) = advance_backoff_counter((COUNTER)); \ + } while (0); + +#define PAUSE_ADAPTIVE_COUNTER(COUNTER) \ + do { \ + (COUNTER) = pause_backoff_counter((COUNTER)); \ + } while (0); +#endif + +#ifdef ENABLE_SPECIALIZATION +/* Multiple threads may execute these concurrently if thread-local bytecode is + * disabled and they all execute the main copy of the bytecode. Specialization + * is disabled in that case so the value is unused, but the RMW cycle should be + * free of data races. + */ +#define RECORD_BRANCH_TAKEN(bitset, flag) \ + FT_ATOMIC_STORE_UINT16_RELAXED( \ + bitset, (FT_ATOMIC_LOAD_UINT16_RELAXED(bitset) << 1) | (flag)) +#else +#define RECORD_BRANCH_TAKEN(bitset, flag) +#endif + +#define UNBOUNDLOCAL_ERROR_MSG \ + "cannot access local variable '%s' where it is not associated with a value" +#define UNBOUNDFREE_ERROR_MSG \ + "cannot access free variable '%s' where it is not associated with a value" \ + " in enclosing scope" +#define NAME_ERROR_MSG "name '%.200s' is not defined" + +// If a trace function sets a new f_lineno and +// *then* raises, we use the destination when searching +// for an exception handler, displaying the traceback, and so on +#define INSTRUMENTED_JUMP(src, dest, event) \ +do { \ + _Py_CODEUNIT *_dest = (dest); \ + if (tstate->tracing) {\ + next_instr = _dest; \ + } else { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + next_instr = _Py_call_instrumentation_jump(this_instr, tstate, event, frame, src, _dest); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + if (next_instr == NULL) { \ + next_instr = _dest + 1; \ + JUMP_TO_LABEL(error); \ + } \ + } \ +} while (0); + + +static inline int _Py_EnterRecursivePy(PyThreadState *tstate) { + return (tstate->py_recursion_remaining-- <= 0) && + _Py_CheckRecursiveCallPy(tstate); +} + +static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { + tstate->py_recursion_remaining++; +} + +/* Implementation of "macros" that modify the instruction pointer, + * stack pointer, or frame pointer. + * These need to treated differently by tier 1 and 2. + * The Tier 1 version is here; Tier 2 is inlined in ceval.c. */ + +#define LOAD_IP(OFFSET) do { \ + next_instr = frame->instr_ptr + (OFFSET); \ + } while (0) + +/* There's no STORE_IP(), it's inlined by the code generator. */ + +#define LOAD_SP() \ +stack_pointer = _PyFrame_GetStackPointer(frame) + +#define SAVE_SP() \ +_PyFrame_SetStackPointer(frame, stack_pointer) + +/* Tier-switching macros. */ + +#define TIER1_TO_TIER2(EXECUTOR) \ +do { \ + OPT_STAT_INC(traces_executed); \ + next_instr = _Py_jit_entry((EXECUTOR), frame, stack_pointer, tstate); \ + frame = tstate->current_frame; \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + int keep_tracing_bit = (uintptr_t)next_instr & 1; \ + next_instr = (_Py_CODEUNIT *)(((uintptr_t)next_instr) & (~1)); \ + if (next_instr == NULL) { \ + /* gh-140104: The exception handler expects frame->instr_ptr + to after this_instr, not this_instr! */ \ + next_instr = frame->instr_ptr + 1; \ + JUMP_TO_LABEL(error); \ + } \ + if (keep_tracing_bit) { \ + assert(uop_buffer_length(&((_PyThreadStateImpl *)tstate)->jit_tracer_state->code_buffer)); \ + ENTER_TRACING(); \ + DISPATCH_NON_TRACING(); \ + } \ + DISPATCH(); \ +} while (0) + +#define TIER2_TO_TIER2(EXECUTOR) \ +do { \ + OPT_STAT_INC(traces_executed); \ + current_executor = (EXECUTOR); \ + goto tier2_start; \ +} while (0) + +#define GOTO_TIER_ONE_SETUP \ + tstate->current_executor = NULL; \ + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \ + _PyFrame_SetStackPointer(frame, stack_pointer); + +#define GOTO_TIER_ONE(TARGET) \ + do \ + { \ + GOTO_TIER_ONE_SETUP \ + return (_Py_CODEUNIT *)(TARGET); \ + } while (0) + +#define GOTO_TIER_ONE_CONTINUE_TRACING(TARGET) \ + do \ + { \ + GOTO_TIER_ONE_SETUP \ + return (_Py_CODEUNIT *)(((uintptr_t)(TARGET))| 1); \ + } while (0) + +#define CURRENT_OPARG() (next_uop[-1].oparg) +#define CURRENT_OPERAND0_64() (next_uop[-1].operand0) +#define CURRENT_OPERAND1_64() (next_uop[-1].operand1) +#define CURRENT_OPERAND0_32() (next_uop[-1].operand0) +#define CURRENT_OPERAND1_32() (next_uop[-1].operand1) +#define CURRENT_OPERAND0_16() (next_uop[-1].operand0) +#define CURRENT_OPERAND1_16() (next_uop[-1].operand1) +#define CURRENT_TARGET() (next_uop[-1].target) + +#define JUMP_TO_JUMP_TARGET() goto jump_to_jump_target +#define JUMP_TO_ERROR() goto jump_to_error_target + +/* Stackref macros */ + +/* How much scratch space to give stackref to PyObject* conversion. */ +#define MAX_STACKREF_SCRATCH 10 + +#define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \ + /* +1 because vectorcall might use -1 to write self */ \ + PyObject *NAME##_temp[MAX_STACKREF_SCRATCH+1]; \ + PyObject **NAME = _PyObjectArray_FromStackRefArray(ARGS, ARG_COUNT, NAME##_temp); + +#define STACKREFS_TO_PYOBJECTS_CLEANUP(NAME) \ + /* +1 because we +1 previously */ \ + _PyObjectArray_Free(NAME - 1, NAME##_temp); + +#define CONVERSION_FAILED(NAME) ((NAME) == NULL) + +#if defined(Py_DEBUG) && !defined(_Py_JIT) +#define SET_CURRENT_CACHED_VALUES(N) current_cached_values = (N) +#define CHECK_CURRENT_CACHED_VALUES(N) assert(current_cached_values == (N)) +#else +#define SET_CURRENT_CACHED_VALUES(N) ((void)0) +#define CHECK_CURRENT_CACHED_VALUES(N) ((void)0) +#endif + +#define IS_PEP523_HOOKED(tstate) (tstate->interp->eval_frame != NULL) + +static inline int +check_periodics(PyThreadState *tstate) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + return _Py_HandlePending(tstate); + } + return 0; +} + +static inline int +check_periodics_at_end(PyThreadState *tstate, _PyInterpreterFrame *frame) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + // Do not handle pending interrupts if the previous instruction was LOAD_SPECIAL + // This may also not handle interrupts if a cache looks like LOAD_SPECIAL, + // but this is benign as we won't skip periodic checks indefinitely. + if (frame->instr_ptr[-1].op.code == LOAD_SPECIAL) { + return 0; + } + return _Py_HandlePending(tstate); + } + return 0; +} + +// Mark the generator as executing. Returns true if the state was changed, +// false if it was already executing or finished. +static inline bool +gen_try_set_executing(PyGenObject *gen) +{ +#ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { + int8_t frame_state = _Py_atomic_load_int8_relaxed(&gen->gi_frame_state); + while (frame_state < FRAME_SUSPENDED_YIELD_FROM_LOCKED) { + if (_Py_atomic_compare_exchange_int8(&gen->gi_frame_state, + &frame_state, + FRAME_EXECUTING)) { + return true; + } + } + // NB: We return false for FRAME_SUSPENDED_YIELD_FROM_LOCKED as well. + // That case is rare enough that we can just handle it in the deopt. + return false; + } +#endif + // Use faster non-atomic modifications in the GIL-enabled build and when + // the object is uniquely referenced in the free-threaded build. + if (gen->gi_frame_state < FRAME_EXECUTING) { + assert(gen->gi_frame_state != FRAME_SUSPENDED_YIELD_FROM_LOCKED); + gen->gi_frame_state = FRAME_EXECUTING; + return true; + } + return false; +} + +// Macro for inplace float binary ops (tier 2 only). +// Mutates the uniquely-referenced TARGET operand in place. +// TARGET must be either left or right. +#define FLOAT_INPLACE_OP(left, right, TARGET, OP) \ + do { \ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); \ + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); \ + assert(PyFloat_CheckExact(left_o)); \ + assert(PyFloat_CheckExact(right_o)); \ + assert(_PyObject_IsUniquelyReferenced( \ + PyStackRef_AsPyObjectBorrow(TARGET))); \ + STAT_INC(BINARY_OP, hit); \ + double _dres = \ + ((PyFloatObject *)left_o)->ob_fval \ + OP ((PyFloatObject *)right_o)->ob_fval; \ + ((PyFloatObject *)PyStackRef_AsPyObjectBorrow(TARGET)) \ + ->ob_fval = _dres; \ + } while (0) + +// Inplace float true division. Sets _divop_err to 1 on zero division. +// Caller must check _divop_err and call ERROR_NO_POP() if set. +#define FLOAT_INPLACE_DIVOP(left, right, TARGET) \ + int _divop_err = 0; \ + do { \ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); \ + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); \ + assert(PyFloat_CheckExact(left_o)); \ + assert(PyFloat_CheckExact(right_o)); \ + assert(_PyObject_IsUniquelyReferenced( \ + PyStackRef_AsPyObjectBorrow(TARGET))); \ + STAT_INC(BINARY_OP, hit); \ + double _divisor = ((PyFloatObject *)right_o)->ob_fval; \ + if (_divisor == 0.0) { \ + PyErr_SetString(PyExc_ZeroDivisionError, \ + "float division by zero"); \ + _divop_err = 1; \ + break; \ + } \ + double _dres = ((PyFloatObject *)left_o)->ob_fval / _divisor; \ + ((PyFloatObject *)PyStackRef_AsPyObjectBorrow(TARGET)) \ + ->ob_fval = _dres; \ + } while (0) + +// Inplace compact int operation. TARGET is expected to be uniquely +// referenced at the optimizer level, but at runtime it may be a +// cached small int singleton. We check _Py_IsImmortal on TARGET +// to decide whether inplace mutation is safe. +// +// After the macro, _int_inplace_res holds the result (may be NULL +// on allocation failure). On success, TARGET was mutated in place +// and _int_inplace_res is a DUP'd reference to it. On fallback +// (small int target, small int result, or overflow), _int_inplace_res +// is from FUNC (_PyCompactLong_Add etc.). +// FUNC is the fallback function (_PyCompactLong_Add etc.) +#define INT_INPLACE_OP(left, right, TARGET, OP, FUNC) \ + _PyStackRef _int_inplace_res = PyStackRef_NULL; \ + do { \ + PyObject *target_o = PyStackRef_AsPyObjectBorrow(TARGET); \ + if (_Py_IsImmortal(target_o)) { \ + break; \ + } \ + assert(_PyObject_IsUniquelyReferenced(target_o)); \ + Py_ssize_t left_val = _PyLong_CompactValue( \ + (PyLongObject *)PyStackRef_AsPyObjectBorrow(left)); \ + Py_ssize_t right_val = _PyLong_CompactValue( \ + (PyLongObject *)PyStackRef_AsPyObjectBorrow(right)); \ + Py_ssize_t result = left_val OP right_val; \ + if (!_PY_IS_SMALL_INT(result) \ + && ((twodigits)((stwodigits)result) + PyLong_MASK \ + < (twodigits)PyLong_MASK + PyLong_BASE)) \ + { \ + _PyLong_SetSignAndDigitCount( \ + (PyLongObject *)target_o, result < 0 ? -1 : 1, 1); \ + ((PyLongObject *)target_o)->long_value.ob_digit[0] = \ + (digit)(result < 0 ? -result : result); \ + _int_inplace_res = PyStackRef_DUP(TARGET); \ + break; \ + } \ + } while (0); \ + if (PyStackRef_IsNull(_int_inplace_res)) { \ + _int_inplace_res = FUNC( \ + (PyLongObject *)PyStackRef_AsPyObjectBorrow(left), \ + (PyLongObject *)PyStackRef_AsPyObjectBorrow(right)); \ + } + +#define CALL_TP_ITERITEM_NO_ESCAPE(ITER, INDEX) \ + Py_TYPE(ITER)->_tp_iteritem((ITER), (INDEX)) diff --git a/cinderx/Interpreter/3.16/Includes/ceval_macros.h b/cinderx/Interpreter/3.16/Includes/ceval_macros.h new file mode 100644 index 000000000..30da39770 --- /dev/null +++ b/cinderx/Interpreter/3.16/Includes/ceval_macros.h @@ -0,0 +1,683 @@ +// Macros and other things needed by ceval.c, and bytecodes.c + +/* Computed GOTOs, or + the-optimization-commonly-but-improperly-known-as-"threaded code" + using gcc's labels-as-values extension + (http://gcc.gnu.org/onlinedocs/gcc/Labels-as-Values.html). + + The traditional bytecode evaluation loop uses a "switch" statement, which + decent compilers will optimize as a single indirect branch instruction + combined with a lookup table of jump addresses. However, since the + indirect jump instruction is shared by all opcodes, the CPU will have a + hard time making the right prediction for where to jump next (actually, + it will be always wrong except in the uncommon case of a sequence of + several identical opcodes). + + "Threaded code" in contrast, uses an explicit jump table and an explicit + indirect jump instruction at the end of each opcode. Since the jump + instruction is at a different address for each opcode, the CPU will make a + separate prediction for each of these instructions, which is equivalent to + predicting the second opcode of each opcode pair. These predictions have + a much better chance to turn out valid, especially in small bytecode loops. + + A mispredicted branch on a modern CPU flushes the whole pipeline and + can cost several CPU cycles (depending on the pipeline depth), + and potentially many more instructions (depending on the pipeline width). + A correctly predicted branch, however, is nearly free. + + At the time of this writing, the "threaded code" version is up to 15-20% + faster than the normal "switch" version, depending on the compiler and the + CPU architecture. + + NOTE: care must be taken that the compiler doesn't try to "optimize" the + indirect jumps by sharing them between all opcodes. Such optimizations + can be disabled on gcc by using the -fno-gcse flag (or possibly + -fno-crossjumping). +*/ + +/* Use macros rather than inline functions, to make it as clear as possible + * to the C compiler that the tracing check is a simple test then branch. + * We want to be sure that the compiler knows this before it generates + * the CFG. + */ + +#ifdef WITH_DTRACE +#define OR_DTRACE_LINE | (PyDTrace_LINE_ENABLED() ? 255 : 0) +#else +#define OR_DTRACE_LINE +#endif + +#ifdef HAVE_COMPUTED_GOTOS + #ifndef USE_COMPUTED_GOTOS + #define USE_COMPUTED_GOTOS 1 + #endif +#else + #if defined(USE_COMPUTED_GOTOS) && USE_COMPUTED_GOTOS + #error "Computed gotos are not supported on this compiler." + #endif + #undef USE_COMPUTED_GOTOS + #define USE_COMPUTED_GOTOS 0 +#endif + +#ifdef Py_STATS +#define INSTRUCTION_STATS(op) \ + do { \ + PyStats *s = _PyStats_GET(); \ + OPCODE_EXE_INC(op); \ + if (s) s->opcode_stats[lastopcode].pair_count[op]++; \ + lastopcode = op; \ + } while (0) +#else +#define INSTRUCTION_STATS(op) ((void)0) +#endif + +#ifdef Py_STATS +# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, const void *instruction_funcptr_table, int oparg, int lastopcode, bool adaptive_enabled +# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg, lastopcode, adaptive_enabled +#else +# define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, const void *instruction_funcptr_table, int oparg, bool adaptive_enabled +# define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg, adaptive_enabled +#endif + +#if _Py_TAIL_CALL_INTERP +# if defined(__clang__) || defined(__GNUC__) +# if !_Py__has_attribute(preserve_none) || !_Py__has_attribute(musttail) +# error "This compiler does not have support for efficient tail calling." +# endif +# elif defined(_MSC_VER) && (_MSC_VER < 1950) +# error "You need at least VS 2026 / PlatformToolset v145 for tail calling." +# endif +# if defined(_MSC_VER) && !defined(__clang__) +# define Py_MUSTTAIL [[msvc::musttail]] +# define Py_PRESERVE_NONE_CC __preserve_none +# else +# define Py_MUSTTAIL __attribute__((musttail)) +# define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) +# endif + typedef PyObject *(Py_PRESERVE_NONE_CC *py_tail_call_funcptr)(TAIL_CALL_PARAMS); + +# define DISPATCH_TABLE_VAR instruction_funcptr_table +# define DISPATCH_TABLE instruction_funcptr_handler_table +# define TRACING_DISPATCH_TABLE instruction_funcptr_tracing_table +# define TARGET(op) Py_NO_INLINE PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_##op(TAIL_CALL_PARAMS) + +# define DISPATCH_GOTO() \ + do { \ + Py_MUSTTAIL return (((py_tail_call_funcptr *)instruction_funcptr_table)[opcode])(TAIL_CALL_ARGS); \ + } while (0) +# define DISPATCH_GOTO_NON_TRACING() \ + do { \ + Py_MUSTTAIL return (((py_tail_call_funcptr *)DISPATCH_TABLE)[opcode])(TAIL_CALL_ARGS); \ + } while (0) +# define JUMP_TO_LABEL(name) \ + do { \ + Py_MUSTTAIL return (_TAIL_CALL_##name)(TAIL_CALL_ARGS); \ + } while (0) +# ifdef Py_STATS +# define JUMP_TO_PREDICTED(name) \ + do { \ + Py_MUSTTAIL return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, this_instr, instruction_funcptr_table, oparg, lastopcode, adaptive_enabled); \ + } while (0) +# else +# define JUMP_TO_PREDICTED(name) \ + do { \ + Py_MUSTTAIL return (_TAIL_CALL_##name)(frame, stack_pointer, tstate, this_instr, instruction_funcptr_table, oparg, adaptive_enabled); \ + } while (0) +# endif +# define LABEL(name) TARGET(name) +#elif USE_COMPUTED_GOTOS +# define DISPATCH_TABLE_VAR opcode_targets +# define DISPATCH_TABLE opcode_targets_table +# define TRACING_DISPATCH_TABLE opcode_tracing_targets_table +# define TARGET(op) TARGET_##op: +# define DISPATCH_GOTO() goto *opcode_targets[opcode] +# define DISPATCH_GOTO_NON_TRACING() goto *DISPATCH_TABLE[opcode]; +# define JUMP_TO_LABEL(name) goto name; +# define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; +# define LABEL(name) name: +#else +# define TARGET(op) case op: TARGET_##op: +# define DISPATCH_GOTO() dispatch_code = opcode | tracing_mode ; goto dispatch_opcode +# define DISPATCH_GOTO_NON_TRACING() dispatch_code = opcode; goto dispatch_opcode +# define JUMP_TO_LABEL(name) goto name; +# define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; +# define LABEL(name) name: +#endif + +#if (_Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS) && _Py_TIER2 +# define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) +# define ENTER_TRACING() \ + DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; +# define LEAVE_TRACING() \ + DISPATCH_TABLE_VAR = DISPATCH_TABLE; +#else +# define IS_JIT_TRACING() (tracing_mode != 0) +# define ENTER_TRACING() tracing_mode = 255 +# define LEAVE_TRACING() tracing_mode = 0 +#endif + +#if _Py_TIER2 +#define STOP_TRACING() \ + do { \ + if (IS_JIT_TRACING()) { \ + LEAVE_TRACING(); \ + _PyJit_FinalizeTracing(tstate, 0); \ + } \ + } while (0); +#else +#define STOP_TRACING() ((void)(0)); +#endif + +/* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ +#ifdef Py_DEBUG +#define PRE_DISPATCH_GOTO() if (frame->lltrace >= 5) { \ + lltrace_instruction(frame, stack_pointer, next_instr, opcode, oparg); } +#else +#define PRE_DISPATCH_GOTO() ((void)0) +#endif + +#ifdef Py_DEBUG +#define LLTRACE_RESUME_FRAME() \ +do { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + frame->lltrace = lltrace; \ +} while (0) +#else +#define LLTRACE_RESUME_FRAME() ((void)0) +#endif + +#ifdef Py_GIL_DISABLED +#define QSBR_QUIESCENT_STATE(tstate) _Py_qsbr_quiescent_state(((_PyThreadStateImpl *)tstate)->qsbr) +#else +#define QSBR_QUIESCENT_STATE(tstate) +#endif + + +/* Do interpreter dispatch accounting for tracing and instrumentation */ +#define DISPATCH() \ + { \ + _PyFrame_StackAssertInvalid(frame); \ + NEXTOPARG(); \ + PRE_DISPATCH_GOTO(); \ + DISPATCH_GOTO(); \ + } + +#define DISPATCH_NON_TRACING() \ + { \ + _PyFrame_StackAssertInvalid(frame); \ + NEXTOPARG(); \ + PRE_DISPATCH_GOTO(); \ + DISPATCH_GOTO_NON_TRACING(); \ + } + +#define DISPATCH_SAME_OPARG() \ + { \ + opcode = next_instr->op.code; \ + PRE_DISPATCH_GOTO(); \ + DISPATCH_GOTO_NON_TRACING(); \ + } + +#define DISPATCH_INLINED(NEW_FRAME) \ + do { \ + assert(!IS_PEP523_HOOKED(tstate)); \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + _PyFrame_StackPointerValidate(frame); \ + assert((NEW_FRAME)->previous == frame); \ + frame = tstate->current_frame = (NEW_FRAME); \ + CALL_STAT_INC(inlined_py_calls); \ + JUMP_TO_LABEL(start_frame); \ + } while (0) + +/* Tuple access macros */ + +#ifndef Py_DEBUG +#define GETITEM(v, i) PyTuple_GET_ITEM((v), (i)) +#else +static inline PyObject * +GETITEM(PyObject *v, Py_ssize_t i) { + assert(PyTuple_Check(v)); + assert(i >= 0); + assert(i < PyTuple_GET_SIZE(v)); + return PyTuple_GET_ITEM(v, i); +} +#endif + +/* Code access macros */ + +/* The integer overflow is checked by an assertion below. */ +#define INSTR_OFFSET() ((int)(next_instr - _PyFrame_GetBytecode(frame))) +#define NEXTOPARG() do { \ + _Py_CODEUNIT word = {.cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t*)next_instr)}; \ + opcode = word.op.code; \ + oparg = word.op.arg; \ + } while (0) + +/* JUMPBY makes the generator identify the instruction as a jump. SKIP_OVER is + * for advancing to the next instruction, taking into account cache entries + * and skipped instructions. + */ +#define JUMPBY(x) (next_instr += (x)) +#define SKIP_OVER(x) (next_instr += (x)) + +#define STACK_LEVEL() ((int)(stack_pointer - _PyFrame_Stackbase(frame))) +#define STACK_SIZE() (_PyFrame_GetCode(frame)->co_stacksize) + +#define WITHIN_STACK_BOUNDS() \ + (frame->owner == FRAME_OWNED_BY_INTERPRETER || (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE())) + +#if defined(Py_DEBUG) && !defined(_Py_JIT) +// This allows temporary stack "overflows", provided it's all in the cache at any point of time. +#define ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(F, L) \ + assert(frame->owner == FRAME_OWNED_BY_INTERPRETER || (STACK_LEVEL() >= 0 && (STACK_LEVEL()) <= STACK_SIZE())) +#else +#define ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE ASSERT_WITHIN_STACK_BOUNDS +#endif + +/* Data access macros */ +#define FRAME_CO_CONSTS (_PyFrame_GetCode(frame)->co_consts) +#define FRAME_CO_NAMES (_PyFrame_GetCode(frame)->co_names) + +/* Local variable macros */ + +#define LOCALS_ARRAY (frame->localsplus) +#define GETLOCAL(i) (frame->localsplus[i]) + + +#ifdef Py_STATS +#define UPDATE_MISS_STATS(INSTNAME) \ + do { \ + STAT_INC(opcode, miss); \ + STAT_INC((INSTNAME), miss); \ + /* The counter is always the first cache entry: */ \ + if (ADAPTIVE_COUNTER_TRIGGERS(next_instr->cache)) { \ + STAT_INC((INSTNAME), deopt); \ + } \ + } while (0) +#else +#define UPDATE_MISS_STATS(INSTNAME) ((void)0) +#endif + + +// Try to lock an object in the free threading build, if it's not already +// locked. Use with a DEOPT_IF() to deopt if the object is already locked. +// These are no-ops in the default GIL build. The general pattern is: +// +// DEOPT_IF(!LOCK_OBJECT(op)); +// if (/* condition fails */) { +// UNLOCK_OBJECT(op); +// DEOPT_IF(true); +// } +// ... +// UNLOCK_OBJECT(op); +// +// NOTE: The object must be unlocked on every exit code path and you should +// avoid any potentially escaping calls (like PyStackRef_CLOSE) while the +// object is locked. +#ifdef Py_GIL_DISABLED +# define LOCK_OBJECT(op) PyMutex_LockFast(&(_PyObject_CAST(op))->ob_mutex) +# define UNLOCK_OBJECT(op) PyMutex_Unlock(&(_PyObject_CAST(op))->ob_mutex) +#else +# define LOCK_OBJECT(op) (1) +# define UNLOCK_OBJECT(op) ((void)0) +#endif + +#define GLOBALS() frame->f_globals +#define BUILTINS() frame->f_builtins +#define LOCALS() frame->f_locals +#define CONSTS() _PyFrame_GetCode(frame)->co_consts +#define NAMES() _PyFrame_GetCode(frame)->co_names + +#if defined(WITH_DTRACE) && !defined(Py_BUILD_CORE_MODULE) +static void dtrace_function_entry(_PyInterpreterFrame *); +static void dtrace_function_return(_PyInterpreterFrame *); + +#define DTRACE_FUNCTION_ENTRY() \ + if (PyDTrace_FUNCTION_ENTRY_ENABLED()) { \ + dtrace_function_entry(frame); \ + } + +#define DTRACE_FUNCTION_RETURN() \ + if (PyDTrace_FUNCTION_RETURN_ENABLED()) { \ + dtrace_function_return(frame); \ + } +#else +#define DTRACE_FUNCTION_ENTRY() ((void)0) +#define DTRACE_FUNCTION_RETURN() ((void)0) +#endif + +/* This takes a uint16_t instead of a _Py_BackoffCounter, + * because it is used directly on the cache entry in generated code, + * which is always an integral type. */ +// Force re-specialization when tracing a side exit to get good side exits. +#define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \ + backoff_counter_triggers(forge_backoff_counter((COUNTER))) + +#ifdef Py_GIL_DISABLED +/* Counters are unreachable when thread-local bytecode is disabled, + * so there is no need to update them. */ +#define ADVANCE_ADAPTIVE_COUNTER(COUNTER) \ + do { \ + _Py_BackoffCounter cnt = (COUNTER); \ + if (!backoff_counter_is_unreachable(cnt)) { \ + (COUNTER) = advance_backoff_counter(cnt); \ + } \ + } while (0); + +#define PAUSE_ADAPTIVE_COUNTER(COUNTER) \ + do { \ + _Py_BackoffCounter cnt = (COUNTER); \ + if (!backoff_counter_is_unreachable(cnt)) { \ + (COUNTER) = pause_backoff_counter(cnt); \ + } \ + } while (0); +#else +#define ADVANCE_ADAPTIVE_COUNTER(COUNTER) \ + do { \ + (COUNTER) = advance_backoff_counter((COUNTER)); \ + } while (0); + +#define PAUSE_ADAPTIVE_COUNTER(COUNTER) \ + do { \ + (COUNTER) = pause_backoff_counter((COUNTER)); \ + } while (0); +#endif + +#ifdef ENABLE_SPECIALIZATION +/* Multiple threads may execute these concurrently if thread-local bytecode is + * disabled and they all execute the main copy of the bytecode. Specialization + * is disabled in that case so the value is unused, but the RMW cycle should be + * free of data races. + */ +#define RECORD_BRANCH_TAKEN(bitset, flag) \ + FT_ATOMIC_STORE_UINT16_RELAXED( \ + bitset, (FT_ATOMIC_LOAD_UINT16_RELAXED(bitset) << 1) | (flag)) +#else +#define RECORD_BRANCH_TAKEN(bitset, flag) +#endif + +#define UNBOUNDLOCAL_ERROR_MSG \ + "cannot access local variable '%s' where it is not associated with a value" +#define UNBOUNDFREE_ERROR_MSG \ + "cannot access free variable '%s' where it is not associated with a value" \ + " in enclosing scope" +#define NAME_ERROR_MSG "name '%.200s' is not defined" + +// If a trace function sets a new f_lineno and +// *then* raises, we use the destination when searching +// for an exception handler, displaying the traceback, and so on +#define INSTRUMENTED_JUMP(src, dest, event) \ +do { \ + _Py_CODEUNIT *_dest = (dest); \ + if (tstate->tracing) {\ + next_instr = _dest; \ + } else { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + next_instr = _Py_call_instrumentation_jump(this_instr, tstate, event, frame, src, _dest); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + if (next_instr == NULL) { \ + next_instr = _dest + 1; \ + JUMP_TO_LABEL(error); \ + } \ + } \ +} while (0); + + +static inline int _Py_EnterRecursivePy(PyThreadState *tstate) { + return (tstate->py_recursion_remaining-- <= 0) && + _Py_CheckRecursiveCallPy(tstate); +} + +static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { + tstate->py_recursion_remaining++; +} + +/* Implementation of "macros" that modify the instruction pointer, + * stack pointer, or frame pointer. + * These need to treated differently by tier 1 and 2. + * The Tier 1 version is here; Tier 2 is inlined in ceval.c. */ + +#define LOAD_IP(OFFSET) do { \ + next_instr = frame->instr_ptr + (OFFSET); \ + } while (0) + +/* There's no STORE_IP(), it's inlined by the code generator. */ + +#define LOAD_SP() \ +stack_pointer = _PyFrame_GetStackPointer(frame) + +#define SAVE_SP() \ +_PyFrame_SetStackPointer(frame, stack_pointer) + +/* Tier-switching macros. */ + +#define TIER1_TO_TIER2(EXECUTOR) \ +do { \ + OPT_STAT_INC(traces_executed); \ + next_instr = _Py_jit_entry((EXECUTOR), frame, stack_pointer, tstate); \ + frame = tstate->current_frame; \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + int keep_tracing_bit = (uintptr_t)next_instr & 1; \ + next_instr = (_Py_CODEUNIT *)(((uintptr_t)next_instr) & (~1)); \ + if (next_instr == NULL) { \ + /* gh-140104: The exception handler expects frame->instr_ptr + to after this_instr, not this_instr! */ \ + next_instr = frame->instr_ptr + 1; \ + JUMP_TO_LABEL(error); \ + } \ + if (keep_tracing_bit) { \ + assert(uop_buffer_length(&((_PyThreadStateImpl *)tstate)->jit_tracer_state->code_buffer)); \ + ENTER_TRACING(); \ + DISPATCH_NON_TRACING(); \ + } \ + DISPATCH(); \ +} while (0) + +#define TIER2_TO_TIER2(EXECUTOR) \ +do { \ + OPT_STAT_INC(traces_executed); \ + current_executor = (EXECUTOR); \ + goto tier2_start; \ +} while (0) + +#define GOTO_TIER_ONE_SETUP \ + tstate->current_executor = NULL; \ + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \ + _PyFrame_SetStackPointer(frame, stack_pointer); + +#define GOTO_TIER_ONE(TARGET) \ + do \ + { \ + GOTO_TIER_ONE_SETUP \ + return (_Py_CODEUNIT *)(TARGET); \ + } while (0) + +#define GOTO_TIER_ONE_CONTINUE_TRACING(TARGET) \ + do \ + { \ + GOTO_TIER_ONE_SETUP \ + return (_Py_CODEUNIT *)(((uintptr_t)(TARGET))| 1); \ + } while (0) + +#define CURRENT_OPARG() (next_uop[-1].oparg) +#define CURRENT_OPERAND0_64() (next_uop[-1].operand0) +#define CURRENT_OPERAND1_64() (next_uop[-1].operand1) +#define CURRENT_OPERAND0_32() (next_uop[-1].operand0) +#define CURRENT_OPERAND1_32() (next_uop[-1].operand1) +#define CURRENT_OPERAND0_16() (next_uop[-1].operand0) +#define CURRENT_OPERAND1_16() (next_uop[-1].operand1) +#define CURRENT_TARGET() (next_uop[-1].target) + +#define JUMP_TO_JUMP_TARGET() goto jump_to_jump_target +#define JUMP_TO_ERROR() goto jump_to_error_target + +/* Stackref macros */ + +/* How much scratch space to give stackref to PyObject* conversion. */ +#define MAX_STACKREF_SCRATCH 10 + +#define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \ + /* +1 because vectorcall might use -1 to write self */ \ + PyObject *NAME##_temp[MAX_STACKREF_SCRATCH+1]; \ + PyObject **NAME = _PyObjectArray_FromStackRefArray(ARGS, ARG_COUNT, NAME##_temp); + +#define STACKREFS_TO_PYOBJECTS_CLEANUP(NAME) \ + /* +1 because we +1 previously */ \ + _PyObjectArray_Free(NAME - 1, NAME##_temp); + +#define CONVERSION_FAILED(NAME) ((NAME) == NULL) + +#if defined(Py_DEBUG) && !defined(_Py_JIT) +#define SET_CURRENT_CACHED_VALUES(N) current_cached_values = (N) +#define CHECK_CURRENT_CACHED_VALUES(N) assert(current_cached_values == (N)) +#else +#define SET_CURRENT_CACHED_VALUES(N) ((void)0) +#define CHECK_CURRENT_CACHED_VALUES(N) ((void)0) +#endif + +#define IS_PEP523_HOOKED(tstate) (tstate->interp->eval_frame != NULL) + +static inline int +check_periodics(PyThreadState *tstate) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + return _Py_HandlePending(tstate); + } + return 0; +} + +static inline int +check_periodics_at_end(PyThreadState *tstate, _PyInterpreterFrame *frame) { + _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); + QSBR_QUIESCENT_STATE(tstate); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + // Do not handle pending interrupts if the previous instruction was LOAD_SPECIAL + // This may also not handle interrupts if a cache looks like LOAD_SPECIAL, + // but this is benign as we won't skip periodic checks indefinitely. + if (frame->instr_ptr[-1].op.code == LOAD_SPECIAL) { + return 0; + } + return _Py_HandlePending(tstate); + } + return 0; +} + +// Mark the generator as executing. Returns true if the state was changed, +// false if it was already executing or finished. +static inline bool +gen_try_set_executing(PyGenObject *gen) +{ +#ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { + int8_t frame_state = _Py_atomic_load_int8_relaxed(&gen->gi_frame_state); + while (frame_state < FRAME_SUSPENDED_YIELD_FROM_LOCKED) { + if (_Py_atomic_compare_exchange_int8(&gen->gi_frame_state, + &frame_state, + FRAME_EXECUTING)) { + return true; + } + } + // NB: We return false for FRAME_SUSPENDED_YIELD_FROM_LOCKED as well. + // That case is rare enough that we can just handle it in the deopt. + return false; + } +#endif + // Use faster non-atomic modifications in the GIL-enabled build and when + // the object is uniquely referenced in the free-threaded build. + if (gen->gi_frame_state < FRAME_EXECUTING) { + assert(gen->gi_frame_state != FRAME_SUSPENDED_YIELD_FROM_LOCKED); + gen->gi_frame_state = FRAME_EXECUTING; + return true; + } + return false; +} + +// Macro for inplace float binary ops (tier 2 only). +// Mutates the uniquely-referenced TARGET operand in place. +// TARGET must be either left or right. +#define FLOAT_INPLACE_OP(left, right, TARGET, OP) \ + do { \ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); \ + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); \ + assert(PyFloat_CheckExact(left_o)); \ + assert(PyFloat_CheckExact(right_o)); \ + assert(_PyObject_IsUniquelyReferenced( \ + PyStackRef_AsPyObjectBorrow(TARGET))); \ + STAT_INC(BINARY_OP, hit); \ + double _dres = \ + ((PyFloatObject *)left_o)->ob_fval \ + OP ((PyFloatObject *)right_o)->ob_fval; \ + ((PyFloatObject *)PyStackRef_AsPyObjectBorrow(TARGET)) \ + ->ob_fval = _dres; \ + } while (0) + +// Inplace float true division. Sets _divop_err to 1 on zero division. +// Caller must check _divop_err and call ERROR_NO_POP() if set. +#define FLOAT_INPLACE_DIVOP(left, right, TARGET) \ + int _divop_err = 0; \ + do { \ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); \ + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); \ + assert(PyFloat_CheckExact(left_o)); \ + assert(PyFloat_CheckExact(right_o)); \ + assert(_PyObject_IsUniquelyReferenced( \ + PyStackRef_AsPyObjectBorrow(TARGET))); \ + STAT_INC(BINARY_OP, hit); \ + double _divisor = ((PyFloatObject *)right_o)->ob_fval; \ + if (_divisor == 0.0) { \ + PyErr_SetString(PyExc_ZeroDivisionError, \ + "float division by zero"); \ + _divop_err = 1; \ + break; \ + } \ + double _dres = ((PyFloatObject *)left_o)->ob_fval / _divisor; \ + ((PyFloatObject *)PyStackRef_AsPyObjectBorrow(TARGET)) \ + ->ob_fval = _dres; \ + } while (0) + +// Inplace compact int operation. TARGET is expected to be uniquely +// referenced at the optimizer level, but at runtime it may be a +// cached small int singleton. We check _Py_IsImmortal on TARGET +// to decide whether inplace mutation is safe. +// +// After the macro, _int_inplace_res holds the result (may be NULL +// on allocation failure). On success, TARGET was mutated in place +// and _int_inplace_res is a DUP'd reference to it. On fallback +// (small int target, small int result, or overflow), _int_inplace_res +// is from FUNC (_PyCompactLong_Add etc.). +// FUNC is the fallback function (_PyCompactLong_Add etc.) +#define INT_INPLACE_OP(left, right, TARGET, OP, FUNC) \ + _PyStackRef _int_inplace_res = PyStackRef_NULL; \ + do { \ + PyObject *target_o = PyStackRef_AsPyObjectBorrow(TARGET); \ + if (_Py_IsImmortal(target_o)) { \ + break; \ + } \ + assert(_PyObject_IsUniquelyReferenced(target_o)); \ + Py_ssize_t left_val = _PyLong_CompactValue( \ + (PyLongObject *)PyStackRef_AsPyObjectBorrow(left)); \ + Py_ssize_t right_val = _PyLong_CompactValue( \ + (PyLongObject *)PyStackRef_AsPyObjectBorrow(right)); \ + Py_ssize_t result = left_val OP right_val; \ + if (!_PY_IS_SMALL_INT(result) \ + && ((twodigits)((stwodigits)result) + PyLong_MASK \ + < (twodigits)PyLong_MASK + PyLong_BASE)) \ + { \ + _PyLong_SetSignAndDigitCount( \ + (PyLongObject *)target_o, result < 0 ? -1 : 1, 1); \ + ((PyLongObject *)target_o)->long_value.ob_digit[0] = \ + (digit)(result < 0 ? -result : result); \ + _int_inplace_res = PyStackRef_DUP(TARGET); \ + break; \ + } \ + } while (0); \ + if (PyStackRef_IsNull(_int_inplace_res)) { \ + _int_inplace_res = FUNC( \ + (PyLongObject *)PyStackRef_AsPyObjectBorrow(left), \ + (PyLongObject *)PyStackRef_AsPyObjectBorrow(right)); \ + } + +#define CALL_TP_ITERITEM_NO_ESCAPE(ITER, INDEX) \ + Py_TYPE(ITER)->_tp_iteritem((ITER), (INDEX)) diff --git a/cinderx/Interpreter/3.16/Includes/generated_cases.c.h b/cinderx/Interpreter/3.16/Includes/generated_cases.c.h new file mode 100644 index 000000000..7d2329f75 --- /dev/null +++ b/cinderx/Interpreter/3.16/Includes/generated_cases.c.h @@ -0,0 +1,15511 @@ +// @generated +// This file is generated by Tools/cases_generator/tier1_generator.py +// from: +// Python/bytecodes.c, 3.16/cinder-bytecodes.c +// Do not edit! + +#ifdef TIER_TWO + #error "This file is for Tier 1 only" +#endif +#define TIER_ONE 1 + +#if !_Py_TAIL_CALL_INTERP +#if !USE_COMPUTED_GOTOS + dispatch_opcode: + switch (dispatch_code) +#endif + { +#endif /* _Py_TAIL_CALL_INTERP */ + /* BEGIN INSTRUCTIONS */ + + + TARGET(BINARY_OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP); + PREDICTED_BINARY_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 6; + (void)this_instr; + _PyStackRef lhs; + _PyStackRef rhs; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef value; + // _SPECIALIZE_BINARY_OP + { + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(BINARY_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + assert(NB_ADD <= oparg); + assert(oparg <= NB_OPARG_LAST); + } + /* Skip 4 cache entries */ + // _BINARY_OP + { + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); + assert(_PyEval_BinaryOps[oparg]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = lhs; + r = rhs; + } + // _POP_TOP + { + value = r; + stack_pointer[-2] = res; + stack_pointer[-1] = l; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = l; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(BINARY_OP_ADD_FLOAT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_ADD_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_ADD_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + } + // _POP_TOP_FLOAT + { + value = r; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + } + // _POP_TOP_FLOAT + { + value = l; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BINARY_OP_ADD_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_ADD_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_ADD_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + l = left; + r = right; + } + // _POP_TOP_INT + { + value = r; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP_INT + { + value = l; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BINARY_OP_ADD_UNICODE) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_ADD_UNICODE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_ADD_UNICODE + { + right = value; + left = nos; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + } + // _POP_TOP_UNICODE + { + value = r; + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + } + // _POP_TOP_UNICODE + { + value = l; + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BINARY_OP_EXTEND) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_EXTEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_EXTEND); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_BINARY_OP_EXTEND + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *descr = read_obj(&this_instr[2].cache); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + assert(d != NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int match = (d->guard != NULL) + ? d->guard(left_o, right_o) + : (Py_TYPE(left_o) == d->lhs_type && Py_TYPE(right_o) == d->rhs_type); + _PyFrame_StackPointerInvalidate(frame); + if (!match) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip -4 cache entry */ + // _BINARY_OP_EXTEND + { + PyObject *descr = read_obj(&this_instr[2].cache); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + STAT_INC(BINARY_OP, hit); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = d->action(left_o, right_o); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + assert(d->result_type == NULL || Py_TYPE(res_o) == d->result_type); + assert(!d->result_unique || Py_REFCNT(res_o) == 1 || _Py_IsImmortal(res_o)); + assert(!PyFloat_CheckExact(res_o) || Py_REFCNT(res_o) == 1); + res = PyStackRef_FromPyObjectSteal(res_o); + l = left; + r = right; + } + // _POP_TOP + { + value = r; + stack_pointer[-2] = res; + stack_pointer[-1] = l; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = l; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_INPLACE_ADD_UNICODE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_INPLACE_ADD_UNICODE + { + right = value; + left = nos; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right))); + int next_oparg; + #if TIER_ONE + assert(next_instr->op.code == STORE_FAST); + next_oparg = next_instr->op.arg; + #else + next_oparg = (int)CURRENT_OPERAND0_16(); + #endif + _PyStackRef *target_local = &GETLOCAL(next_oparg); + assert(PyUnicode_CheckExact(left_o)); + if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + assert(Py_REFCNT(left_o) >= 2 || !PyStackRef_IsHeapSafe(left)); + PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); + PyObject *right_o = PyStackRef_AsPyObjectSteal(right); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyUnicode_Append(&temp, right_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); + _PyFrame_StackPointerInvalidate(frame); + *target_local = PyStackRef_NULL; + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(temp); + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BINARY_OP_MULTIPLY_FLOAT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_MULTIPLY_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_MULTIPLY_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + } + // _POP_TOP_FLOAT + { + value = r; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + } + // _POP_TOP_FLOAT + { + value = l; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BINARY_OP_MULTIPLY_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_MULTIPLY_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_MULTIPLY_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + l = left; + r = right; + } + // _POP_TOP_INT + { + value = r; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP_INT + { + value = l; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BINARY_OP_SUBSCR_DICT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef nos; + _PyStackRef dict_st; + _PyStackRef sub_st; + _PyStackRef res; + _PyStackRef ds; + _PyStackRef ss; + _PyStackRef value; + // _GUARD_NOS_DICT_SUBSCRIPT + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!Py_TYPE(o)->tp_as_mapping) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (Py_TYPE(o)->tp_as_mapping->mp_subscript != _PyDict_Subscript) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_DICT + { + sub_st = stack_pointer[-1]; + dict_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(Py_TYPE(dict)->tp_as_mapping->mp_subscript == _PyDict_Subscript); + STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = _PyDict_Subscript(dict, sub); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + ds = dict_st; + ss = sub_st; + } + // _POP_TOP + { + value = ss; + stack_pointer[-2] = res; + stack_pointer[-1] = ds; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = ds; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(BINARY_OP_SUBSCR_GETITEM) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_GETITEM; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef container; + _PyStackRef getitem; + _PyStackRef sub; + _PyStackRef new_frame; + /* Skip 5 cache entries */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _BINARY_OP_SUBSCR_CHECK_FUNC + { + container = stack_pointer[-2]; + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); + if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); + if (getitem_o == NULL) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + assert(PyFunction_Check(getitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); + if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); + assert(code->co_argcount == 2); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + getitem = PyStackRef_FromPyObjectNew(getitem_o); + } + // _BINARY_OP_SUBSCR_INIT_CALL + { + sub = stack_pointer[-1]; + STAT_INC(BINARY_OP, hit); + _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + pushed_frame->localsplus[0] = container; + pushed_frame->localsplus[1] = sub; + frame->return_offset = 6u ; + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + CI_UPDATE_CALL_COUNT + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(BINARY_OP_SUBSCR_LIST_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_LIST_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef list_st; + _PyStackRef sub_st; + _PyStackRef res; + _PyStackRef ls; + _PyStackRef ss; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_LIST + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_LIST_INT + { + sub_st = value; + list_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + if (index < 0) { + index += PyList_GET_SIZE(list); + } + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + res = PyStackRef_FromPyObjectSteal(res_o); + #else + if (index < 0 || index >= PyList_GET_SIZE(list)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(BINARY_OP, hit); + ls = list_st; + ss = sub_st; + } + // _POP_TOP_INT + { + value = ss; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP + { + value = ls; + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(BINARY_OP_SUBSCR_LIST_SLICE) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_LIST_SLICE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_SLICE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef nos; + _PyStackRef list_st; + _PyStackRef sub_st; + _PyStackRef res; + _PyStackRef ls; + _PyStackRef ss; + _PyStackRef value; + // _GUARD_TOS_SLICE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PySlice_Check(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_LIST + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_LIST_SLICE + { + sub_st = tos; + list_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PySlice_Check(sub)); + assert(PyList_CheckExact(list)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = _PyList_SliceSubscript(list, sub); + _PyFrame_StackPointerInvalidate(frame); + STAT_INC(BINARY_OP, hit); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + ls = list_st; + ss = sub_st; + } + // _POP_TOP + { + value = ss; + stack_pointer[-2] = res; + stack_pointer[-1] = ls; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = ls; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(BINARY_OP_SUBSCR_STR_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_STR_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef str_st; + _PyStackRef sub_st; + _PyStackRef res; + _PyStackRef s; + _PyStackRef i; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_COMPACT_ASCII + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + if (!PyUnicode_IS_COMPACT_ASCII(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_STR_INT + { + sub_st = value; + str_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); + assert(PyLong_CheckExact(sub)); + assert(PyUnicode_CheckExact(str)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (PyUnicode_GET_LENGTH(str) <= index) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + uint8_t c = PyUnicode_1BYTE_DATA(str)[index]; + assert(c < 128); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + s = str_st; + i = sub_st; + res = PyStackRef_FromPyObjectBorrow(res_o); + } + // _POP_TOP_INT + { + value = i; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP_UNICODE + { + value = s; + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BINARY_OP_SUBSCR_TUPLE_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_TUPLE_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef tuple_st; + _PyStackRef sub_st; + _PyStackRef res; + _PyStackRef ts; + _PyStackRef ss; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_TUPLE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyTuple_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS + { + sub_st = value; + tuple_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (index >= PyTuple_GET_SIZE(tuple)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_TUPLE_INT + { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); + STAT_INC(BINARY_OP, hit); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + res = PyStackRef_FromPyObjectNew(res_o); + ts = tuple_st; + ss = sub_st; + } + // _POP_TOP_INT + { + value = ss; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP + { + value = ts; + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(BINARY_OP_SUBSCR_USTR_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_USTR_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_USTR_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef str_st; + _PyStackRef sub_st; + _PyStackRef res; + _PyStackRef s; + _PyStackRef i; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_USTR_INT + { + sub_st = value; + str_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); + assert(PyLong_CheckExact(sub)); + assert(PyUnicode_CheckExact(str)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (PyUnicode_GET_LENGTH(str) <= index) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + s = str_st; + i = sub_st; + res = PyStackRef_FromPyObjectBorrow(res_o); + } + // _POP_TOP_INT + { + value = i; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP_UNICODE + { + value = s; + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BINARY_OP_SUBTRACT_FLOAT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBTRACT_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBTRACT_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + PyObject *d = PyFloat_FromDouble(dres); + if (d == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(d); + l = left; + r = right; + } + // _POP_TOP_FLOAT + { + value = r; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + } + // _POP_TOP_FLOAT + { + value = l; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BINARY_OP_SUBTRACT_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBTRACT_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBTRACT_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + l = left; + r = right; + } + // _POP_TOP_INT + { + value = r; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP_INT + { + value = l; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BINARY_SLICE) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_SLICE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BINARY_SLICE); + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + _PyStackRef res; + // _SPECIALIZE_BINARY_SLICE + { + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(BINARY_SLICE); + #endif /* ENABLE_SPECIALIZATION */ + } + // _BINARY_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); + PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); + PyObject *res_o; + if (PyList_CheckExact(container_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + res_o = _PyList_BinarySlice(container_o, start_o, stop_o); + _PyFrame_StackPointerInvalidate(frame); + } + else if (PyTuple_CheckExact(container_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + res_o = _PyTuple_BinarySlice(container_o, start_o, stop_o); + _PyFrame_StackPointerInvalidate(frame); + } + else if (PyUnicode_CheckExact(container_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + res_o = _PyUnicode_BinarySlice(container_o, start_o, stop_o); + _PyFrame_StackPointerInvalidate(frame); + } + else { + PyObject *slice = PySlice_New(start_o, stop_o, NULL); + if (slice == NULL) { + res_o = NULL; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + res_o = PyObject_GetItem(container_o, slice); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(slice); + _PyFrame_StackPointerInvalidate(frame); + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = stop; + stop = PyStackRef_NULL; + stack_pointer[-1] = stop; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = start; + start = PyStackRef_NULL; + stack_pointer[-2] = start; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-3] = container; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BUILD_INTERPOLATION) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_INTERPOLATION; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_INTERPOLATION); + _PyStackRef value; + _PyStackRef str; + _PyStackRef *format; + _PyStackRef interpolation; + format = &stack_pointer[-(oparg & 1)]; + str = stack_pointer[-1 - (oparg & 1)]; + value = stack_pointer[-2 - (oparg & 1)]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + PyObject *str_o = PyStackRef_AsPyObjectBorrow(str); + int conversion = oparg >> 2; + PyObject *format_o; + if (oparg & 1) { + format_o = PyStackRef_AsPyObjectBorrow(format[0]); + } + else { + format_o = &_Py_STR(empty); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *interpolation_o = _PyInterpolation_Build(value_o, str_o, conversion, format_o); + _PyFrame_StackPointerInvalidate(frame); + if (oparg & 1) { + stack_pointer += -(oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(format[0]); + _PyFrame_StackPointerInvalidate(frame); + } + else { + stack_pointer += -(oparg & 1); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(str); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + if (interpolation_o == NULL) { + JUMP_TO_LABEL(error); + } + interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); + stack_pointer[0] = interpolation; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BUILD_LIST) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_LIST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_LIST); + _PyStackRef *values; + _PyStackRef list; + values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); + _PyFrame_StackPointerInvalidate(frame); + if (list_o == NULL) { + JUMP_TO_LABEL(error); + } + list = PyStackRef_FromPyObjectStealMortal(list_o); + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BUILD_MAP) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_MAP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_MAP); + _PyStackRef *values; + _PyStackRef map; + values = &stack_pointer[-oparg*2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *map_o = _Py_BuildMap_StackRefSteal(values, oparg); + _PyFrame_StackPointerInvalidate(frame); + if (map_o == NULL) { + stack_pointer += -oparg*2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + map = PyStackRef_FromPyObjectStealMortal(map_o); + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BUILD_SET) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_SET; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SET); + _PyStackRef *values; + _PyStackRef set; + values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *set_o = PySet_New(NULL); + _PyFrame_StackPointerInvalidate(frame); + if (set_o == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + int err = 0; + for (Py_ssize_t i = 0; i < oparg; i++) { + _PyStackRef value = values[i]; + values[i] = PyStackRef_NULL; + if (err == 0) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(value)); + _PyFrame_StackPointerInvalidate(frame); + } + else { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + } + if (err) { + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(set_o); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + set = PyStackRef_FromPyObjectStealMortal(set_o); + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BUILD_SLICE) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_SLICE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SLICE); + _PyStackRef *args; + _PyStackRef slice; + args = &stack_pointer[-oparg]; + PyObject *start_o = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); + PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; + PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (slice_o == NULL) { + JUMP_TO_LABEL(error); + } + slice = PyStackRef_FromPyObjectStealMortal(slice_o); + stack_pointer[0] = slice; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BUILD_STRING) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_STRING; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_STRING); + _PyStackRef *pieces; + _PyStackRef str; + pieces = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *str_o = _Py_BuildString_StackRefSteal(pieces, oparg); + _PyFrame_StackPointerInvalidate(frame); + if (str_o == NULL) { + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + str = PyStackRef_FromPyObjectSteal(str_o); + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BUILD_TEMPLATE) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_TEMPLATE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TEMPLATE); + _PyStackRef strings; + _PyStackRef interpolations; + _PyStackRef template; + interpolations = stack_pointer[-1]; + strings = stack_pointer[-2]; + PyObject *strings_o = PyStackRef_AsPyObjectBorrow(strings); + PyObject *interpolations_o = PyStackRef_AsPyObjectBorrow(interpolations); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(interpolations); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(strings); + _PyFrame_StackPointerInvalidate(frame); + if (template_o == NULL) { + JUMP_TO_LABEL(error); + } + template = PyStackRef_FromPyObjectSteal(template_o); + stack_pointer[0] = template; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(BUILD_TUPLE) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_TUPLE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TUPLE); + _PyStackRef *values; + _PyStackRef tup; + values = &stack_pointer[-oparg]; + PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); + if (tup_o == NULL) { + JUMP_TO_LABEL(error); + } + tup = PyStackRef_FromPyObjectStealMortal(tup_o); + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(CACHE) { + #if _Py_TAIL_CALL_INTERP + int opcode = CACHE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CACHE); + assert(0 && "Executing a cache."); + Py_FatalError("Executing a cache."); + DISPATCH(); + } + + TARGET(CALL) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL); + PREDICTED_CALL:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + opcode = CALL; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + // _SPECIALIZE_CALL + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CALL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD + { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + } + // _DO_CALL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (Py_TYPE(callable_o) == &PyFunction_Type && + !IS_PEP523_HOOKED(tstate) && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + frame->return_offset = 4u ; + DISPATCH_INLINED(new_frame); + } + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( + callable, + arguments, + total_args, + PyStackRef_NULL, + opcode == INSTRUMENTED_CALL, + frame, + this_instr, + tstate); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics_at_end(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_ALLOC_AND_ENTER_INIT) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_ALLOC_AND_ENTER_INIT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef init; + _PyStackRef self; + _PyStackRef *args; + _PyStackRef init_frame; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_OBJECT + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyStackRef_IsNull(self_or_null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyType_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyTypeObject *tp = (PyTypeObject *)callable_o; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _ALLOCATE_OBJECT + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(self_or_null)); + assert(PyType_Check(callable_o)); + PyTypeObject *tp = (PyTypeObject *)callable_o; + assert(tp->tp_new == PyBaseObject_Type.tp_new); + assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_alloc == PyType_GenericAlloc); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; + PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); + PyCodeObject *code = (PyCodeObject *)init_func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *self_o = PyType_GenericAlloc(tp, 0); + _PyFrame_StackPointerInvalidate(frame); + if (self_o == NULL) { + JUMP_TO_LABEL(error); + } + self_or_null = PyStackRef_FromPyObjectSteal(self_o); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(init_func); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // _CREATE_INIT_FRAME + { + args = &stack_pointer[-oparg]; + self = self_or_null; + init = callable; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + _PyFrame_StackPointerInvalidate(frame); + assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); + shim->localsplus[0] = PyStackRef_DUP(self); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, init, NULL, args-1, oparg+1, NULL, shim); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, shim); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + tstate->py_recursion_remaining--; + init_frame = PyStackRef_Wrap(temp); + } + // _PUSH_FRAME + { + new_frame = init_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + CI_UPDATE_CALL_COUNT + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BOUND_METHOD_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + STAT_INC(CALL, hit); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _CHECK_FUNCTION_VERSION + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + CI_UPDATE_CALL_COUNT + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(CALL_BOUND_METHOD_GENERAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BOUND_METHOD_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_METHOD_VERSION + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _EXPAND_METHOD + { + self_or_null = null; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(self_or_null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + CI_UPDATE_CALL_COUNT + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(CALL_BUILTIN_CLASS) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_CLASS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_CALLABLE_BUILTIN_CLASS + { + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyType_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyTypeObject *tp = (PyTypeObject *)callable_o; + if (tp->tp_vectorcall == NULL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_BUILTIN_CLASS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = _Py_CallBuiltinClass_StackRef( + callable, + arguments, + total_args); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP_OPARG + { + args = &stack_pointer[-oparg]; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef_CloseStack(args, oparg); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = self_or_null; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _CHECK_PERIODIC_AT_END + { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics_at_end(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_BUILTIN_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_CALLABLE_BUILTIN_FAST + { + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_BUILTIN_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = _Py_BuiltinCallFast_StackRef( + callable, + arguments, + total_args + ); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP_OPARG + { + args = &stack_pointer[-oparg]; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef_CloseStack(args, oparg); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = self_or_null; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _CHECK_PERIODIC_AT_END + { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics_at_end(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS + { + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_BUILTIN_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRef(callable, arguments, total_args); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP_OPARG + { + args = &stack_pointer[-oparg]; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef_CloseStack(args, oparg); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = self_or_null; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _CHECK_PERIODIC_AT_END + { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics_at_end(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_BUILTIN_O) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_O; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + _PyStackRef c; + _PyStackRef s; + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_CALLABLE_BUILTIN_O + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_LIMIT + { + if (_Py_ReachedRecursionLimit(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_BUILTIN_O + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyStackRef_IsNull(self_or_null)) { + args--; + } + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + _PyStackRef arg = args[0]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); + _PyFrame_StackPointerInvalidate(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + c = callable; + s = args[0]; + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _POP_TOP + { + value = s; + stack_pointer[-2 - oparg] = res; + stack_pointer[-1 - oparg] = c; + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = c; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _CHECK_PERIODIC_AT_END + { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics_at_end(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_EX_NON_PY_GENERAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_EX_NON_PY_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CALL_EX_NON_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX == 1, "incorrect cache size"); + _PyStackRef func_st; + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef null; + _PyStackRef callargs_st; + _PyStackRef kwargs_st; + _PyStackRef result; + /* Skip 1 cache entry */ + // _CHECK_IS_NOT_PY_CALLABLE_EX + { + func_st = stack_pointer[-4]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + if (Py_TYPE(func) == &PyFunction_Type && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + UPDATE_MISS_STATS(CALL_FUNCTION_EX); + assert(_PyOpcode_Deopt[opcode] == (CALL_FUNCTION_EX)); + JUMP_TO_PREDICTED(CALL_FUNCTION_EX); + } + } + // _MAKE_CALLARGS_A_TUPLE + { + callargs = stack_pointer[-2]; + func = func_st; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (!PyTuple_CheckExact(callargs_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + _PyFrame_StackPointerInvalidate(frame); + if (tuple_o == NULL) { + JUMP_TO_LABEL(error); + } + _PyStackRef temp = callargs; + callargs = PyStackRef_FromPyObjectSteal(tuple_o); + stack_pointer[-2] = callargs; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + } + // _CALL_FUNCTION_EX_NON_PY_GENERAL + { + kwargs_st = stack_pointer[-1]; + callargs_st = callargs; + null = stack_pointer[-3]; + func_st = func; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + (void)null; + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *result_o = PyObject_Call(func, callargs, kwargs); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(kwargs_st); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(callargs_st); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(func_st); + _PyFrame_StackPointerInvalidate(frame); + if (result_o == NULL) { + JUMP_TO_LABEL(error); + } + result = PyStackRef_FromPyObjectSteal(result_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = result; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics_at_end(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_EX_PY) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_EX_PY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CALL_EX_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX == 1, "incorrect cache size"); + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef func_st; + _PyStackRef callargs_st; + _PyStackRef kwargs_st; + _PyStackRef ex_frame; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL_FUNCTION_EX); + assert(_PyOpcode_Deopt[opcode] == (CALL_FUNCTION_EX)); + JUMP_TO_PREDICTED(CALL_FUNCTION_EX); + } + } + // _MAKE_CALLARGS_A_TUPLE + { + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (!PyTuple_CheckExact(callargs_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + _PyFrame_StackPointerInvalidate(frame); + if (tuple_o == NULL) { + JUMP_TO_LABEL(error); + } + _PyStackRef temp = callargs; + callargs = PyStackRef_FromPyObjectSteal(tuple_o); + stack_pointer[-2] = callargs; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + } + // _CHECK_IS_PY_CALLABLE_EX + { + func_st = func; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + if (Py_TYPE(func) != &PyFunction_Type) { + UPDATE_MISS_STATS(CALL_FUNCTION_EX); + assert(_PyOpcode_Deopt[opcode] == (CALL_FUNCTION_EX)); + JUMP_TO_PREDICTED(CALL_FUNCTION_EX); + } + if (((PyFunctionObject *)func)->vectorcall != _PyFunction_Vectorcall) { + UPDATE_MISS_STATS(CALL_FUNCTION_EX); + assert(_PyOpcode_Deopt[opcode] == (CALL_FUNCTION_EX)); + JUMP_TO_PREDICTED(CALL_FUNCTION_EX); + } + } + // _PY_FRAME_EX + { + kwargs_st = stack_pointer[-1]; + callargs_st = callargs; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + assert(Py_TYPE(func) == &PyFunction_Type); + assert(((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + ex_frame = PyStackRef_Wrap(new_frame); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + new_frame = ex_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + CI_UPDATE_CALL_COUNT + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(CALL_FUNCTION_EX) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_FUNCTION_EX; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CALL_FUNCTION_EX); + PREDICTED_CALL_FUNCTION_EX:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = CALL_FUNCTION_EX; + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef func_st; + _PyStackRef null; + _PyStackRef callargs_st; + _PyStackRef kwargs_st; + _PyStackRef result; + // _SPECIALIZE_CALL_FUNCTION_EX + { + func = stack_pointer[-4]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_CallFunctionEx(func, next_instr); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CALL_FUNCTION_EX); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + // _MAKE_CALLARGS_A_TUPLE + { + callargs = stack_pointer[-2]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (!PyTuple_CheckExact(callargs_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + _PyFrame_StackPointerInvalidate(frame); + if (tuple_o == NULL) { + JUMP_TO_LABEL(error); + } + _PyStackRef temp = callargs; + callargs = PyStackRef_FromPyObjectSteal(tuple_o); + stack_pointer[-2] = callargs; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + } + // _DO_CALL_FUNCTION_EX + { + kwargs_st = stack_pointer[-1]; + callargs_st = callargs; + null = stack_pointer[-3]; + func_st = func; + (void)null; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + _PyFrame_StackPointerInvalidate(frame); + if (err) { + JUMP_TO_LABEL(error); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + result_o = PyObject_Call(func, callargs, kwargs); + _PyFrame_StackPointerInvalidate(frame); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result_o == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + _PyFrame_StackPointerInvalidate(frame); + } + else { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_CLEAR(result_o); + _PyFrame_StackPointerInvalidate(frame); + } + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + !IS_PEP523_HOOKED(tstate) && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + assert( 2u == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX); + frame->return_offset = 2u ; + DISPATCH_INLINED(new_frame); + } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + result_o = PyObject_Call(func, callargs, kwargs); + _PyFrame_StackPointerInvalidate(frame); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(kwargs_st); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(callargs_st); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(func_st); + _PyFrame_StackPointerInvalidate(frame); + if (result_o == NULL) { + JUMP_TO_LABEL(error); + } + result = PyStackRef_FromPyObjectSteal(result_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = result; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics_at_end(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_INTRINSIC_1) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_INTRINSIC_1; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_1); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + // _CALL_INTRINSIC_1 + { + value = stack_pointer[-1]; + assert(oparg <= MAX_INTRINSIC_1); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + v = value; + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _POP_TOP + { + value = v; + stack_pointer[-1] = res; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(CALL_INTRINSIC_2) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_INTRINSIC_2; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_2); + _PyStackRef value2_st; + _PyStackRef value1_st; + _PyStackRef res; + _PyStackRef vs1; + _PyStackRef vs2; + _PyStackRef value; + // _CALL_INTRINSIC_2 + { + value1_st = stack_pointer[-1]; + value2_st = stack_pointer[-2]; + assert(oparg <= MAX_INTRINSIC_2); + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + vs1 = value1_st; + vs2 = value2_st; + } + // _POP_TOP + { + value = vs2; + stack_pointer[-2] = res; + stack_pointer[-1] = vs1; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = vs1; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(CALL_ISINSTANCE) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_ISINSTANCE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ISINSTANCE); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef instance; + _PyStackRef cls; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_THIRD_NULL + { + null = stack_pointer[-3]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_CALLABLE_ISINSTANCE + { + callable = stack_pointer[-4]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.isinstance) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_ISINSTANCE + { + cls = stack_pointer[-1]; + instance = stack_pointer[-2]; + STAT_INC(CALL, hit); + PyObject *inst_o = PyStackRef_AsPyObjectBorrow(instance); + PyObject *cls_o = PyStackRef_AsPyObjectBorrow(cls); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int retval = PyObject_IsInstance(inst_o, cls_o); + _PyFrame_StackPointerInvalidate(frame); + if (retval < 0) { + JUMP_TO_LABEL(error); + } + (void)null; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(cls); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(instance); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(callable); + _PyFrame_StackPointerInvalidate(frame); + res = retval ? PyStackRef_True : PyStackRef_False; + assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(CALL_KW) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW); + PREDICTED_CALL_KW:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + opcode = CALL_KW; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef res; + // _SPECIALIZE_CALL_KW + { + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CALL_KW); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + } + // _DO_CALL_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + if (Py_TYPE(callable_o) == &PyFunction_Type && + !IS_PEP523_HOOKED(tstate) && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4u ; + DISPATCH_INLINED(new_frame); + } + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( + callable, + arguments, + total_args, + kwnames, + opcode == INSTRUMENTED_CALL_KW, + frame, + this_instr, + tstate); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(CALL_KW_BOUND_METHOD) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW_BOUND_METHOD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _CHECK_METHOD_VERSION_KW + { + null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _EXPAND_METHOD_KW + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + _PyStackRef callable_s = callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(callable_s); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + CI_UPDATE_CALL_COUNT + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(CALL_KW_NON_PY) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW_NON_PY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_NON_PY); + opcode = CALL_KW_NON_PY; + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE_KW + { + callable = stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _CALL_KW_NON_PY + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + kwnames); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics_at_end(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_KW_PY) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW_PY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _CHECK_FUNCTION_VERSION_KW + { + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + CI_UPDATE_CALL_COUNT + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(CALL_LEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_LEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LEN); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + _PyStackRef a; + _PyStackRef c; + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_CALLABLE_LEN + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.len) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_LEN + { + arg = stack_pointer[-1]; + STAT_INC(CALL, hit); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_ssize_t len_i = PyObject_Length(arg_o); + _PyFrame_StackPointerInvalidate(frame); + if (len_i < 0) { + JUMP_TO_LABEL(error); + } + PyObject *res_o = PyLong_FromSsize_t(len_i); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + a = arg; + c = callable; + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _POP_TOP + { + value = c; + stack_pointer[-3] = res; + stack_pointer[-2] = a; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = a; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(CALL_LIST_APPEND) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_LIST_APPEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LIST_APPEND); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef nos; + _PyStackRef self; + _PyStackRef arg; + _PyStackRef none; + _PyStackRef c; + _PyStackRef s; + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_CALLABLE_LIST_APPEND + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.list_append) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_NOS_NOT_NULL + { + nos = stack_pointer[-2]; + if (PyStackRef_IsNull(nos)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_NOS_LIST + { + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_LIST_APPEND + { + arg = stack_pointer[-1]; + self = nos; + assert(oparg == 1); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + if (!LOCK_OBJECT(self_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); + if (err) { + JUMP_TO_LABEL(error); + } + c = callable; + s = self; + none = PyStackRef_None; + } + // _POP_TOP + { + value = s; + stack_pointer[-3] = none; + stack_pointer[-2] = c; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = c; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(CALL_METHOD_DESCRIPTOR_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (method->d_method->ml_flags != METH_FASTCALL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args == 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyObject *self = PyStackRef_AsPyObjectBorrow( + PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_METHOD_DESCRIPTOR_FAST + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + assert(self != NULL); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyCFunctionFast cfunc = _PyCFunctionFast_CAST(method->d_method->ml_meth); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = _PyCallMethodDescriptorFast_StackRef( + callable, + cfunc, + self, + arguments, + total_args + ); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP_OPARG + { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef_CloseStack(args, oparg); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = self_or_null; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _CHECK_PERIODIC_AT_END + { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics_at_end(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (method->d_method->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (total_args == 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + assert(self != NULL); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords_CAST(method->d_method->ml_meth); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRef( + callable, + cfunc, + self, + arguments, + total_args + ); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[-2 - oparg] = callable; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP_OPARG + { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef_CloseStack(args, oparg); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = self_or_null; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _CHECK_PERIODIC_AT_END + { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics_at_end(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_NOARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + _PyStackRef c; + _PyStackRef s; + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (method->d_method->ml_flags != METH_NOARGS) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyObject *self = PyStackRef_AsPyObjectBorrow( + PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_LIMIT + { + if (_Py_ReachedRecursionLimit(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_METHOD_DESCRIPTOR_NOARGS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + assert(oparg == 1 || !PyStackRef_IsNull(self_or_null)); + if (!PyStackRef_IsNull(self_or_null)) { + args--; + } + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); + STAT_INC(CALL, hit); + PyCFunction cfunc = method->d_method->ml_meth; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); + _PyFrame_StackPointerInvalidate(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + c = callable; + s = args[0]; + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _POP_TOP + { + value = s; + stack_pointer[-2 - oparg] = res; + stack_pointer[-1 - oparg] = c; + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = c; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _CHECK_PERIODIC_AT_END + { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics_at_end(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_METHOD_DESCRIPTOR_O) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_O; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + _PyStackRef c; + _PyStackRef s; + _PyStackRef a; + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_CALLABLE_METHOD_DESCRIPTOR_O + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (method->d_method->ml_flags != METH_O) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + total_args++; + } + if (total_args != 2) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyObject *self = PyStackRef_AsPyObjectBorrow( + PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_LIMIT + { + if (_Py_ReachedRecursionLimit(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_METHOD_DESCRIPTOR_O + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + } + STAT_INC(CALL, hit); + PyCFunction cfunc = method->d_method->ml_meth; + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + PyObject *arg = PyStackRef_AsPyObjectBorrow(arguments[1]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, arg); + _PyFrame_StackPointerInvalidate(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + c = callable; + s = arguments[0]; + a = arguments[1]; + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _POP_TOP + { + value = a; + stack_pointer[-2 - oparg] = res; + stack_pointer[-1 - oparg] = c; + stack_pointer[-oparg] = s; + stack_pointer += 1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = s; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = c; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _CHECK_PERIODIC_AT_END + { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics_at_end(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_NON_PY_GENERAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_NON_PY_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_NON_PY_GENERAL); + opcode = CALL_NON_PY_GENERAL; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE + { + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_NON_PY_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + PyStackRef_NULL); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics_at_end(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_PY_EXACT_ARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_PY_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + CI_UPDATE_CALL_COUNT + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(CALL_PY_GENERAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_PY_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + CI_UPDATE_CALL_COUNT + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(CALL_STR_1) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_STR_1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_STR_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + _PyStackRef a; + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_CALLABLE_STR_1 + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyUnicode_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_STR_1 + { + arg = stack_pointer[-1]; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = PyObject_Str(arg_o); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + a = arg; + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _POP_TOP + { + value = a; + stack_pointer[-3] = res; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _CHECK_PERIODIC_AT_END + { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics_at_end(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_TUPLE_1) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_TUPLE_1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TUPLE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + _PyStackRef a; + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_CALLABLE_TUPLE_1 + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyTuple_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_TUPLE_1 + { + arg = stack_pointer[-1]; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = PySequence_Tuple(arg_o); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + a = arg; + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _POP_TOP + { + value = a; + stack_pointer[-3] = res; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _CHECK_PERIODIC_AT_END + { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics_at_end(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(CALL_TYPE_1) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_TYPE_1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TYPE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + _PyStackRef a; + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_CALLABLE_TYPE_1 + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyType_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_TYPE_1 + { + arg = stack_pointer[-1]; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + STAT_INC(CALL, hit); + a = arg; + res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); + } + // _POP_TOP + { + value = a; + stack_pointer[-3] = res; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(CHECK_EG_MATCH) { + #if _Py_TAIL_CALL_INTERP + int opcode = CHECK_EG_MATCH; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EG_MATCH); + _PyStackRef exc_value_st; + _PyStackRef match_type_st; + _PyStackRef rest; + _PyStackRef match; + match_type_st = stack_pointer[-1]; + exc_value_st = stack_pointer[-2]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + PyObject *match_o = NULL; + PyObject *rest_o = NULL; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, + &match_o, &rest_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res < 0) { + JUMP_TO_LABEL(error); + } + assert((match_o == NULL) == (rest_o == NULL)); + if (match_o == NULL) { + JUMP_TO_LABEL(error); + } + if (!Py_IsNone(match_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyErr_SetHandledException(match_o); + _PyFrame_StackPointerInvalidate(frame); + } + rest = PyStackRef_FromPyObjectSteal(rest_o); + match = PyStackRef_FromPyObjectSteal(match_o); + stack_pointer[0] = rest; + stack_pointer[1] = match; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(CHECK_EXC_MATCH) { + #if _Py_TAIL_CALL_INTERP + int opcode = CHECK_EXC_MATCH; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EXC_MATCH); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyExceptionInstance_Check(left_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _PyEval_CheckExceptTypeValid(tstate, right_o); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int res = PyErr_GivenExceptionMatches(left_o, right_o); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(right); + _PyFrame_StackPointerInvalidate(frame); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = b; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(CLEANUP_THROW) { + #if _Py_TAIL_CALL_INTERP + int opcode = CLEANUP_THROW; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CLEANUP_THROW); + _PyStackRef sub_iter; + _PyStackRef null_in; + _PyStackRef last_sent_val; + _PyStackRef exc_value_st; + _PyStackRef none; + _PyStackRef null_out; + _PyStackRef value; + exc_value_st = stack_pointer[-1]; + last_sent_val = stack_pointer[-2]; + null_in = stack_pointer[-3]; + sub_iter = stack_pointer[-4]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + #if !_Py_TAIL_CALL_INTERP + assert(throwflag); + #endif + assert(exc_value && PyExceptionInstance_Check(exc_value)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); + _PyFrame_StackPointerInvalidate(frame); + if (matches) { + value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = sub_iter; + sub_iter = value; + stack_pointer[-4] = sub_iter; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-1] = exc_value_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = last_sent_val; + last_sent_val = PyStackRef_NULL; + stack_pointer[-2] = last_sent_val; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = null_in; + null_in = PyStackRef_NULL; + stack_pointer[-3] = null_in; + PyStackRef_XCLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -4; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + null_out = null_in; + none = PyStackRef_None; + } + else { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + JUMP_TO_LABEL(exception_unwind); + } + stack_pointer[0] = none; + stack_pointer[1] = null_out; + stack_pointer[2] = value; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(COMPARE_OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP); + PREDICTED_COMPARE_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _SPECIALIZE_COMPARE_OP + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(COMPARE_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + // _COMPARE_OP + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + if (oparg & 16) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int res_bool = PyObject_IsTrue(res_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(res_o); + _PyFrame_StackPointerInvalidate(frame); + if (res_bool < 0) { + JUMP_TO_LABEL(error); + } + res = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res = PyStackRef_FromPyObjectSteal(res_o); + } + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(COMPARE_OP_FLOAT) { + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + } + /* Skip 1 cache entry */ + // _COMPARE_OP_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); + int sign_ish = COMPARISON_BIT(dleft, dright); + l = left; + r = right; + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + } + // _POP_TOP_FLOAT + { + value = r; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + } + // _POP_TOP_FLOAT + { + value = l; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(COMPARE_OP_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_INT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + } + /* Skip 1 cache entry */ + // _COMPARE_OP_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(_PyLong_IsCompact((PyLongObject *)left_o)); + assert(_PyLong_IsCompact((PyLongObject *)right_o)); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + _PyLong_DigitCount((PyLongObject *)right_o) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); + int sign_ish = COMPARISON_BIT(ileft, iright); + l = left; + r = right; + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + } + // _POP_TOP_INT + { + value = r; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP_INT + { + value = l; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(COMPARE_OP_STR) { + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP_STR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_STR); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + _PyStackRef l; + _PyStackRef r; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + } + /* Skip 1 cache entry */ + // _COMPARE_OP_STR + { + right = value; + left = nos; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left_o, right_o); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + l = left; + r = right; + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; + } + // _POP_TOP_UNICODE + { + value = r; + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + } + // _POP_TOP_UNICODE + { + value = l; + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(CONTAINS_OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = CONTAINS_OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP); + PREDICTED_CONTAINS_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + _PyStackRef l; + _PyStackRef r; + _PyStackRef value; + // _SPECIALIZE_CONTAINS_OP + { + right = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_ContainsOp(right, next_instr); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CONTAINS_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + // _CONTAINS_OP + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int res = PySequence_Contains(right_o, left_o); + _PyFrame_StackPointerInvalidate(frame); + if (res < 0) { + JUMP_TO_LABEL(error); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + } + // _POP_TOP + { + value = r; + stack_pointer[-2] = b; + stack_pointer[-1] = l; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = l; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(CONTAINS_OP_DICT) { + #if _Py_TAIL_CALL_INTERP + int opcode = CONTAINS_OP_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_DICT); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + _PyStackRef l; + _PyStackRef r; + _PyStackRef value; + // _GUARD_TOS_ANY_DICT + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnyDict_CheckExact(o)) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + JUMP_TO_PREDICTED(CONTAINS_OP); + } + } + /* Skip 1 cache entry */ + // _CONTAINS_OP_DICT + { + right = tos; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyAnyDict_CheckExact(right_o)); + STAT_INC(CONTAINS_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int res = PyDict_Contains(right_o, left_o); + _PyFrame_StackPointerInvalidate(frame); + if (res < 0) { + JUMP_TO_LABEL(error); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + } + // _POP_TOP + { + value = r; + stack_pointer[-2] = b; + stack_pointer[-1] = l; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = l; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(CONTAINS_OP_SET) { + #if _Py_TAIL_CALL_INTERP + int opcode = CONTAINS_OP_SET; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_SET); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + _PyStackRef l; + _PyStackRef r; + _PyStackRef value; + // _GUARD_TOS_ANY_SET + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnySet_CheckExact(o)) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + JUMP_TO_PREDICTED(CONTAINS_OP); + } + } + /* Skip 1 cache entry */ + // _CONTAINS_OP_SET + { + right = tos; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyAnySet_CheckExact(right_o)); + STAT_INC(CONTAINS_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int res = _PySet_Contains((PySetObject *)right_o, left_o); + _PyFrame_StackPointerInvalidate(frame); + if (res < 0) { + JUMP_TO_LABEL(error); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + } + // _POP_TOP + { + value = r; + stack_pointer[-2] = b; + stack_pointer[-1] = l; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = l; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(CONVERT_VALUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = CONVERT_VALUE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONVERT_VALUE); + _PyStackRef value; + _PyStackRef result; + value = stack_pointer[-1]; + conversion_func conv_fn; + assert(oparg >= FVC_STR && oparg <= FVC_ASCII); + conv_fn = _PyEval_ConversionFuncs[oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + if (result_o == NULL) { + JUMP_TO_LABEL(error); + } + result = PyStackRef_FromPyObjectSteal(result_o); + stack_pointer[0] = result; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(COPY) { + #if _Py_TAIL_CALL_INTERP + int opcode = COPY; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY); + _PyStackRef bottom; + _PyStackRef top; + bottom = stack_pointer[-1 - (oparg-1)]; + top = PyStackRef_DUP(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(COPY_FREE_VARS) { + #if _Py_TAIL_CALL_INTERP + int opcode = COPY_FREE_VARS; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY_FREE_VARS); + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); + } + DISPATCH(); + } + + TARGET(DELETE_DEREF) { + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_DEREF; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_DEREF); + PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); + if (oldobj == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(oldobj); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH(); + } + + TARGET(DELETE_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_FAST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_FAST); + _PyStackRef v = GETLOCAL(oparg); + if (PyStackRef_IsNull(v)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH(); + } + + TARGET(DELETE_SUBSCR) { + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_SUBSCR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_SUBSCR); + _PyStackRef container; + _PyStackRef sub; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), + PyStackRef_AsPyObjectBorrow(sub)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (err) { + JUMP_TO_LABEL(error); + } + DISPATCH(); + } + + TARGET(DICT_MERGE) { + #if _Py_TAIL_CALL_INTERP + int opcode = DICT_MERGE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_MERGE); + _PyStackRef callable; + _PyStackRef dict; + _PyStackRef update; + _PyStackRef u; + _PyStackRef value; + // _DICT_MERGE + { + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + PyObject *dupkey = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _PyDict_MergeUniq(dict_o, update_o, &dupkey); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FormatKwargsError(tstate, callable_o, update_o, dupkey); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_XDECREF(dupkey); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + u = update; + } + // _POP_TOP + { + value = u; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(DICT_UPDATE) { + #if _Py_TAIL_CALL_INTERP + int opcode = DICT_UPDATE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_UPDATE); + _PyStackRef dict; + _PyStackRef update; + _PyStackRef upd; + _PyStackRef value; + // _DICT_UPDATE + { + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = PyDict_Update(dict_o, update_o); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + if (matches) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyObject *exc = _PyErr_GetRaisedException(tstate); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int has_keys = PyObject_HasAttrWithError(update_o, &_Py_ID(keys)); + _PyFrame_StackPointerInvalidate(frame); + if (has_keys == 0) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_Format(tstate, PyExc_TypeError, + "'%T' object is not a mapping", + update_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(exc); + _PyFrame_StackPointerInvalidate(frame); + } + else { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_ChainExceptions1(exc); + _PyFrame_StackPointerInvalidate(frame); + } + } + JUMP_TO_LABEL(error); + } + upd = update; + } + // _POP_TOP + { + value = upd; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(END_ASYNC_FOR) { + #if _Py_TAIL_CALL_INTERP + int opcode = END_ASYNC_FOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_ASYNC_FOR); + _PyStackRef awaitable_st; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + awaitable_st = stack_pointer[-2]; + JUMPBY(0); + (void)oparg; + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(exc && PyExceptionInstance_Check(exc)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); + _PyFrame_StackPointerInvalidate(frame); + if (matches) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = exc_st; + exc_st = PyStackRef_NULL; + stack_pointer[-1] = exc_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = awaitable_st; + awaitable_st = PyStackRef_NULL; + stack_pointer[-2] = awaitable_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + } + else { + Py_INCREF(exc); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_SetRaisedException(tstate, exc); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + JUMP_TO_LABEL(exception_unwind); + } + DISPATCH(); + } + + TARGET(END_FOR) { + #if _Py_TAIL_CALL_INTERP + int opcode = END_FOR; + (void)(opcode); + #endif + next_instr += 1; + INSTRUCTION_STATS(END_FOR); + _PyStackRef value; + value = stack_pointer[-1]; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH(); + } + + TARGET(END_SEND) { + #if _Py_TAIL_CALL_INTERP + int opcode = END_SEND; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_SEND); + _PyStackRef receiver; + _PyStackRef index_or_null; + _PyStackRef value; + _PyStackRef val; + value = stack_pointer[-1]; + index_or_null = stack_pointer[-2]; + receiver = stack_pointer[-3]; + val = value; + (void)index_or_null; + stack_pointer[-3] = val; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(receiver); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH(); + } + + TARGET(ENTER_EXECUTOR) { + #if _Py_TAIL_CALL_INTERP + int opcode = ENTER_EXECUTOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(ENTER_EXECUTOR); + opcode = ENTER_EXECUTOR; + #ifdef _Py_TIER2 + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; + if (IS_JIT_TRACING()) { + int og_opcode = executor->vm_data.opcode; + int og_oparg = (oparg & ~255) | executor->vm_data.oparg; + next_instr = this_instr; + if (_PyJit_EnterExecutorShouldStopTracing(og_opcode)) { + if (_PyOpcode_Caches[_PyOpcode_Deopt[og_opcode]]) { + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + opcode = og_opcode; + oparg = og_oparg; + DISPATCH_GOTO_NON_TRACING(); + } + JUMP_TO_LABEL(stop_tracing); + } + assert(executor->vm_data.index == INSTR_OFFSET() - 1); + assert(executor->vm_data.code == code); + assert(executor->vm_data.valid); + assert(tstate->current_executor == NULL); + uintptr_t iversion = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) != iversion) { + opcode = executor->vm_data.opcode; + oparg = (oparg & ~255) | executor->vm_data.oparg; + next_instr = this_instr; + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + DISPATCH_GOTO(); + } + assert(executor != tstate->interp->cold_executor); + tstate->jit_exit = NULL; + TIER1_TO_TIER2(executor); + #else + Py_FatalError("ENTER_EXECUTOR is not supported in this build"); + #endif /* _Py_TIER2 */ + } + + TARGET(EXIT_INIT_CHECK) { + #if _Py_TAIL_CALL_INTERP + int opcode = EXIT_INIT_CHECK; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXIT_INIT_CHECK); + _PyStackRef should_be_none; + should_be_none = stack_pointer[-1]; + if (!PyStackRef_IsNone(should_be_none)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(EXTENDED_ARG) { + #if _Py_TAIL_CALL_INTERP + int opcode = EXTENDED_ARG; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXTENDED_ARG); + opcode = EXTENDED_ARG; + assert(oparg); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); + } + + TARGET(EXTENDED_OPCODE) { + #if _Py_TAIL_CALL_INTERP + int opcode = EXTENDED_OPCODE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXTENDED_OPCODE); + _PyStackRef *args; + _PyStackRef *top; + args = &stack_pointer[-(oparg >> 2)]; + top = &stack_pointer[-(oparg >> 2)]; + int extop = (int)next_instr->op.code; + int extoparg = (int)next_instr->op.arg; + while (extop == EXTENDED_ARG) { + SKIP_OVER(1); + extoparg = extoparg << 8 | next_instr->op.arg; + extop = next_instr->op.code; + } + extop |= EXTENDED_OPCODE_FLAG; + if (extop == PRIMITIVE_LOAD_CONST) { + top[0] = PyStackRef_FromPyObjectNew( + PyTuple_GET_ITEM(GETITEM(FRAME_CO_CONSTS, extoparg), 0)); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + } else if (extop == STORE_LOCAL) { + _PyStackRef val = args[0]; + PyObject* local = GETITEM(FRAME_CO_CONSTS, extoparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int index = PyLong_AsInt(PyTuple_GET_ITEM(local, 0)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int type = + _PyClassLoader_ResolvePrimitiveType(PyTuple_GET_ITEM(local, 1)); + _PyFrame_StackPointerInvalidate(frame); + if (type < 0) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + _PyStackRef tmp = GETLOCAL(index); + if (type == TYPED_DOUBLE) { + GETLOCAL(index) = PyStackRef_DUP(val); + } else { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_ssize_t ival = + unbox_primitive_int(PyStackRef_AsPyObjectBorrow(val)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + GETLOCAL(index) = + PyStackRef_FromPyObjectSteal(box_primitive(type, ival)); + _PyFrame_StackPointerInvalidate(frame); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + #if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) + if (adaptive_enabled) { + if (index < INT8_MAX && type < INT8_MAX) { + int16_t* cache = (int16_t*)next_instr; + *cache = (index << 8) | type; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _Ci_specialize(next_instr, STORE_LOCAL_CACHED); + _PyFrame_StackPointerInvalidate(frame); + } + } + #endif + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + } else if (extop == LOAD_LOCAL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int index = PyLong_AsInt( + PyTuple_GET_ITEM(GETITEM(FRAME_CO_CONSTS, extoparg), 0)); + _PyFrame_StackPointerInvalidate(frame); + _PyStackRef value = GETLOCAL(index); + if (PyStackRef_IsNull(value)) { + GETLOCAL(index) = value = + PyStackRef_FromPyObjectSteal(PyLong_FromLong(0)); + } + value = PyStackRef_DUP(value); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + top[0] = value; + } else if (extop == PRIMITIVE_BOX) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + top[0] = sign_extend_primitive(args[0], extoparg); + _PyFrame_StackPointerInvalidate(frame); + } else if (extop == PRIMITIVE_UNBOX) { + PyObject* val = PyStackRef_AsPyObjectBorrow(args[0]); + if (PyLong_CheckExact(val)) { + size_t value; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int overflow = _PyClassLoader_CheckOverflow(val, extoparg, &value); + _PyFrame_StackPointerInvalidate(frame); + if (!overflow) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyErr_SetString(PyExc_OverflowError, "int overflow"); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + } + } else if (extop == SEQUENCE_GET) { + PyObject* sequence = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject* idx = PyStackRef_AsPyObjectBorrow(args[1]); + PyObject* item; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_ssize_t val = (Py_ssize_t)PyLong_AsVoidPtr(idx); + _PyFrame_StackPointerInvalidate(frame); + if (val == -1 && _PyErr_Occurred(tstate)) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + if (val < 0) { + val += Py_SIZE(sequence); + } + extoparg &= ~SEQ_SUBSCR_UNCHECKED; + if (extoparg == SEQ_LIST) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + item = PyList_GetItem(sequence, val); + _PyFrame_StackPointerInvalidate(frame); + if (item == NULL) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + Py_INCREF(item); + } else if (extoparg == SEQ_LIST_INEXACT) { + if (PyList_CheckExact(sequence) || + Py_TYPE(sequence)->tp_as_sequence->sq_item == + PyList_Type.tp_as_sequence->sq_item) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + item = PyList_GetItem(sequence, val); + _PyFrame_StackPointerInvalidate(frame); + if (item == NULL) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + Py_INCREF(item); + } else { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + item = PyObject_GetItem(sequence, idx); + _PyFrame_StackPointerInvalidate(frame); + if (item == NULL) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + } + } else if (extoparg == SEQ_CHECKED_LIST) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + item = Ci_CheckedList_GetItem(sequence, val); + _PyFrame_StackPointerInvalidate(frame); + if (item == NULL) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + } else if (extoparg == SEQ_ARRAY_INT64) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + item = _Ci_StaticArray_Get(sequence, val); + _PyFrame_StackPointerInvalidate(frame); + if (item == NULL) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + } else { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyErr_Format( + PyExc_SystemError, "bad oparg for SEQUENCE_GET: %d", extoparg); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + top[0] = PyStackRef_FromPyObjectSteal(item); + } else if (extop == SEQUENCE_SET) { + PyObject* v = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject* sequence = PyStackRef_AsPyObjectBorrow(args[1]); + PyObject* subscr = PyStackRef_AsPyObjectBorrow(args[2]); + int err; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_ssize_t idx = (Py_ssize_t)PyLong_AsVoidPtr(subscr); + _PyFrame_StackPointerInvalidate(frame); + if (idx == -1 && _PyErr_Occurred(tstate)) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + if (idx < 0) { + idx += Py_SIZE(sequence); + } + if (extoparg == SEQ_LIST) { + Py_INCREF(v); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + err = PyList_SetItem(sequence, idx, v); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(v); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + } else if (extoparg == SEQ_LIST_INEXACT) { + if (PyList_CheckExact(sequence) || + Py_TYPE(sequence)->tp_as_sequence->sq_ass_item == + PyList_Type.tp_as_sequence->sq_ass_item) { + Py_INCREF(v); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + err = PyList_SetItem(sequence, idx, v); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(v); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + } else { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + err = PyObject_SetItem(sequence, subscr, v); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + } + } else if (extoparg == SEQ_ARRAY_INT64) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + err = _Ci_StaticArray_Set(sequence, idx, v); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + } else { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyErr_Format( + PyExc_SystemError, "bad oparg for SEQUENCE_SET: %d", oparg); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + } else if (extop == FAST_LEN) { + PyObject* collection = PyStackRef_AsPyObjectBorrow(args[0]); + int inexact = extoparg & FAST_LEN_INEXACT; + extoparg &= ~FAST_LEN_INEXACT; + assert(FAST_LEN_LIST <= extoparg && extoparg <= FAST_LEN_STR); + PyObject* length; + if (inexact) { + if ((extoparg == FAST_LEN_LIST && PyList_CheckExact(collection)) || + (extoparg == FAST_LEN_DICT && PyDict_CheckExact(collection)) || + (extoparg == FAST_LEN_SET && PyAnySet_CheckExact(collection)) || + (extoparg == FAST_LEN_TUPLE && PyTuple_CheckExact(collection)) || + (extoparg == FAST_LEN_ARRAY && + PyStaticArray_CheckExact(collection)) || + (extoparg == FAST_LEN_STR && PyUnicode_CheckExact(collection))) { + inexact = 0; + } + } + if (inexact) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_ssize_t res = PyObject_Size(collection); + _PyFrame_StackPointerInvalidate(frame); + length = res >= 0 ? PyLong_FromSsize_t(res) : NULL; + } else if (extoparg == FAST_LEN_DICT) { + if (Ci_CheckedDict_Check(collection)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + length = PyLong_FromLong(PyObject_Size(collection)); + _PyFrame_StackPointerInvalidate(frame); + } else { + assert(PyDict_Check(collection)); + length = PyLong_FromLong(((PyDictObject*)collection)->ma_used); + } + } else if (extoparg == FAST_LEN_SET) { + assert(PyAnySet_Check(collection)); + length = PyLong_FromLong(((PySetObject*)collection)->used); + } else { + assert( + PyTuple_Check(collection) || PyList_Check(collection) || + PyStaticArray_CheckExact(collection) || + PyUnicode_Check(collection) || Ci_CheckedList_Check(collection)); + length = PyLong_FromLong(Py_SIZE(collection)); + } + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + if (length == NULL) { + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + top[0] = PyStackRef_FromPyObjectSteal(length); + } else if (extop == LIST_DEL) { + PyObject* list = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject* subscr = PyStackRef_AsPyObjectBorrow(args[1]); + int err; + Py_ssize_t idx = PyLong_AsLong(subscr); + if (idx == -1 && _PyErr_Occurred(tstate)) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + err = PyList_SetSlice(list, idx, idx + 1, NULL); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + } else if (extop == REFINE_TYPE) { + } else if (extop == LOAD_CLASS) { + PyObject* type_descr = GETITEM(FRAME_CO_CONSTS, extoparg); + int optional; + int exact; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject* type = (PyObject*)_PyClassLoader_ResolveType( + type_descr, &optional, &exact); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + if (type == NULL) { + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + top[0] = PyStackRef_FromPyObjectSteal(type); + } else if (extop == LOAD_TYPE) { + PyObject* instance = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject* type = (PyObject*)Py_TYPE(instance); + Py_INCREF(type); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + top[0] = PyStackRef_FromPyObjectSteal(type); + } else if (extop == BUILD_CHECKED_LIST) { + PyObject* list; + PyObject* list_info = GETITEM(FRAME_CO_CONSTS, extoparg); + PyObject* list_type = PyTuple_GET_ITEM(list_info, 0); + Py_ssize_t list_size = PyLong_AsLong(PyTuple_GET_ITEM(list_info, 1)); + int optional; + int exact; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyTypeObject* type = + _PyClassLoader_ResolveType(list_type, &optional, &exact); + _PyFrame_StackPointerInvalidate(frame); + assert(!optional); + #if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) + if (adaptive_enabled) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + specialize_with_value( + next_instr, (PyObject*)type, BUILD_CHECKED_LIST_CACHED, 0, 0); + _PyFrame_StackPointerInvalidate(frame); + } + #endif + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + list = Ci_CheckedList_New(type, list_size); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(type); + _PyFrame_StackPointerInvalidate(frame); + if (list == NULL) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + for (Py_ssize_t i = 0; i < list_size; i++) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Ci_ListOrCheckedList_SET_ITEM( + list, i, PyStackRef_AsPyObjectBorrow(args[i])); + _PyFrame_StackPointerInvalidate(frame); + } + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + top[0] = PyStackRef_FromPyObjectSteal(list); + } else if (extop == BUILD_CHECKED_MAP) { + PyObject* map_info = GETITEM(FRAME_CO_CONSTS, extoparg); + PyObject* map_type = PyTuple_GET_ITEM(map_info, 0); + Py_ssize_t map_size = PyLong_AsLong(PyTuple_GET_ITEM(map_info, 1)); + int optional; + int exact; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyTypeObject* type = + _PyClassLoader_ResolveType(map_type, &optional, &exact); + _PyFrame_StackPointerInvalidate(frame); + if (type == NULL) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + assert(!optional); + #if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) + if (adaptive_enabled) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + specialize_with_value( + next_instr, (PyObject*)type, BUILD_CHECKED_MAP_CACHED, 0, 0); + _PyFrame_StackPointerInvalidate(frame); + } + #endif + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyObject* map = Ci_CheckedDict_NewPresized(type, map_size); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(type); + _PyFrame_StackPointerInvalidate(frame); + if (map == NULL) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + if (ci_build_dict(args, map_size, map) < 0) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(map); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + top[0] = PyStackRef_FromPyObjectSteal(map); + } else if (extop == LOAD_METHOD_STATIC) { + PyObject* self = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject* value = GETITEM(FRAME_CO_CONSTS, extoparg); + PyObject* target = PyTuple_GET_ITEM(value, 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int is_classmethod = _PyClassLoader_IsClassMethodDescr(value); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_ssize_t slot = _PyClassLoader_ResolveMethod(target); + _PyFrame_StackPointerInvalidate(frame); + if (slot == -1) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + #if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) + if (adaptive_enabled) { + if (slot < (INT32_MAX >> 1)) { + int32_t* cache = (int32_t*)next_instr; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + *cache = load_method_static_cached_oparg(slot, is_classmethod); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _Ci_specialize(next_instr, LOAD_METHOD_STATIC_CACHED); + _PyFrame_StackPointerInvalidate(frame); + } + } + #endif + + _PyType_VTable* vtable; + if (is_classmethod) { + vtable = (_PyType_VTable*)(((PyTypeObject*)self)->tp_cache); + } else { + vtable = (_PyType_VTable*)self->ob_type->tp_cache; + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + assert(!PyErr_Occurred()); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + StaticMethodInfo res = + _PyClassLoader_LoadStaticMethod(vtable, slot, self); + _PyFrame_StackPointerInvalidate(frame); + if (res.lmr_func == NULL) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + _PyStackRef self_ref = PyStackRef_DUP(args[0]); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + top[0] = PyStackRef_FromPyObjectSteal(res.lmr_func); + top[1] = self_ref; + } else if (extop == INVOKE_METHOD) { + PyObject* target = PyStackRef_AsPyObjectBorrow(args[0]); + Py_ssize_t nargs = (oparg >> 2) - 1; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(!PyErr_Occurred()); + _PyFrame_StackPointerInvalidate(frame); + STACKREFS_TO_PYOBJECTS(&args[1], nargs, args_o); + if (CONVERSION_FAILED(args_o)) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyObject* res = PyObject_Vectorcall(target, args_o, nargs, NULL); + _PyFrame_StackPointerInvalidate(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + if (res == NULL) { + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + top[0] = PyStackRef_FromPyObjectSteal(res); + } else if (extop == INVOKE_FUNCTION) { + PyObject* value = GETITEM(FRAME_CO_CONSTS, extoparg); + int nargs = oparg >> 2; + PyObject* target = PyTuple_GET_ITEM(value, 0); + PyObject* container; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject* func = _PyClassLoader_ResolveFunction(target, &container); + _PyFrame_StackPointerInvalidate(frame); + if (func == NULL) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + if (CONVERSION_FAILED(args_o)) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyObject* res = _PyObject_Vectorcall(func, args_o, nargs, NULL); + _PyFrame_StackPointerInvalidate(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + #if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) + if (adaptive_enabled) { + if (_PyClassLoader_IsImmutable(container)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + specialize_with_value( + next_instr, func, INVOKE_FUNCTION_CACHED, 0, 0); + _PyFrame_StackPointerInvalidate(frame); + } else { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyObject** funcptr = _PyClassLoader_ResolveIndirectPtr(target); + _PyFrame_StackPointerInvalidate(frame); + PyObject*** cache = (PyObject***)next_instr; + *cache = funcptr; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _Ci_specialize(next_instr, INVOKE_INDIRECT_CACHED); + _PyFrame_StackPointerInvalidate(frame); + } + } + #endif + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(func); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(container); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + if (res == NULL) { + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + top[0] = PyStackRef_FromPyObjectSteal(res); + } else if (extop == INVOKE_NATIVE) { + PyObject* value = GETITEM(FRAME_CO_CONSTS, extoparg); + assert(PyTuple_CheckExact(value)); + Py_ssize_t nargs = oparg >> 2; + PyObject* target = PyTuple_GET_ITEM(value, 0); + PyObject* name = PyTuple_GET_ITEM(target, 0); + PyObject* symbol = PyTuple_GET_ITEM(target, 1); + PyObject* signature = PyTuple_GET_ITEM(value, 1); + STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + if (CONVERSION_FAILED(args_o)) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject* res = _PyClassloader_InvokeNativeFunction( + name, symbol, signature, args_o, nargs); + _PyFrame_StackPointerInvalidate(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + if (res == NULL) { + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + top[0] = PyStackRef_FromPyObjectSteal(res); + } else if (extop == TP_ALLOC) { + int optional; + int exact; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyTypeObject* type = _PyClassLoader_ResolveType( + GETITEM(FRAME_CO_CONSTS, extoparg), &optional, &exact); + _PyFrame_StackPointerInvalidate(frame); + assert(!optional); + if (type == NULL) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyObject* inst = type->tp_alloc(type, 0); + _PyFrame_StackPointerInvalidate(frame); + #if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) + if (adaptive_enabled) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + specialize_with_value(next_instr, func, TP_ALLOC_CACHED, 0, 0); + _PyFrame_StackPointerInvalidate(frame); + } + #endif + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(type); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + if (inst == NULL) { + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + top[0] = PyStackRef_FromPyObjectSteal(inst); + } else if (extop == CAST) { + PyObject* val = PyStackRef_AsPyObjectBorrow(args[0]); + int optional; + int exact; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyTypeObject* type = _PyClassLoader_ResolveType( + GETITEM(FRAME_CO_CONSTS, extoparg), &optional, &exact); + _PyFrame_StackPointerInvalidate(frame); + if (type == NULL) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + #if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) + if (adaptive_enabled) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + specialize_with_value( + next_instr, + (PyObject*)type, + CAST_CACHED, + 2, + (exact << 1) | optional); + _PyFrame_StackPointerInvalidate(frame); + } + #endif + _PyStackRef res; + if (!_PyObject_TypeCheckOptional(val, type, optional, exact)) { + if (type == &PyFloat_Type && PyObject_TypeCheck(val, &PyLong_Type)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + double dval = PyLong_AsDouble(val); + _PyFrame_StackPointerInvalidate(frame); + if (dval == -1.0 && PyErr_Occurred()) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + PyObject* fval = PyFloat_FromDouble(dval); + if (fval == NULL) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(fval); + } else { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyErr_Format( + PyExc_TypeError, + exact ? "expected exactly '%s', got '%s'" + : "expected '%s', got '%s'", + type->tp_name, + Py_TYPE(val)->tp_name); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(type); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + } else { + res = PyStackRef_FromPyObjectNew(val); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(type); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + top[0] = res; + } else if (extop == PRIMITIVE_UNARY_OP) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject* res = + primitive_unary_op(PyStackRef_AsPyObjectBorrow(args[0]), extoparg); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + if (res == NULL) { + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + top[0] = PyStackRef_FromPyObjectSteal(res); + } else if (extop == PRIMITIVE_BINARY_OP) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject* res = primitive_binary_op( + PyStackRef_AsPyObjectBorrow(args[0]), + PyStackRef_AsPyObjectBorrow(args[1]), + extoparg); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + if (res == NULL) { + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + top[0] = PyStackRef_FromPyObjectSteal(res); + } else if (extop == PRIMITIVE_COMPARE_OP) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject* res = primitive_compare_op( + PyStackRef_AsPyObjectBorrow(args[0]), + PyStackRef_AsPyObjectBorrow(args[1]), + extoparg); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + if (res == NULL) { + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + top[0] = PyStackRef_FromPyObjectSteal(res); + } else if (extop == LOAD_FIELD) { + PyObject* self = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject* field = GETITEM(FRAME_CO_CONSTS, extoparg); + PyObject* value; + int field_type; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_ssize_t offset = + _PyClassLoader_ResolveFieldOffset(field, &field_type); + _PyFrame_StackPointerInvalidate(frame); + if (offset == -1) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + if (field_type == TYPED_OBJECT) { + value = *FIELD_OFFSET(self, offset); + #if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) + if (adaptive_enabled) { + if (offset < INT32_MAX) { + int32_t* cache = (int32_t*)next_instr; + *cache = offset; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _Ci_specialize(next_instr, LOAD_OBJ_FIELD); + _PyFrame_StackPointerInvalidate(frame); + } + } + #endif + + if (value == NULL) { + PyObject* name = + PyTuple_GET_ITEM(field, PyTuple_GET_SIZE(field) - 1); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyErr_Format( + PyExc_AttributeError, + "'%.50s' object has no attribute '%U'", + Py_TYPE(self)->tp_name, + name); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + Py_INCREF(value); + } else { + #if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) + if (adaptive_enabled) { + if (offset <= INT32_MAX >> 8) { + assert(field_type < 0xff); + int32_t* cache = (int32_t*)next_instr; + *cache = offset << 8 | field_type; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _Ci_specialize(next_instr, LOAD_PRIMITIVE_FIELD); + _PyFrame_StackPointerInvalidate(frame); + } + } + #endif + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + value = load_field(field_type, (char*)FIELD_OFFSET(self, offset)); + _PyFrame_StackPointerInvalidate(frame); + if (value == NULL) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + } + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + top[0] = PyStackRef_FromPyObjectSteal(value); + } else if (extop == STORE_FIELD) { + PyObject* value = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject* self = PyStackRef_AsPyObjectBorrow(args[1]); + PyObject* field = GETITEM(FRAME_CO_CONSTS, extoparg); + int field_type; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_ssize_t offset = + _PyClassLoader_ResolveFieldOffset(field, &field_type); + _PyFrame_StackPointerInvalidate(frame); + if (offset == -1) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + PyObject** addr = FIELD_OFFSET(self, offset); + if (field_type == TYPED_OBJECT) { + Py_INCREF(value); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_XDECREF(*addr); + _PyFrame_StackPointerInvalidate(frame); + *addr = value; + #if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) + if (adaptive_enabled) { + if (offset <= INT32_MAX) { + int32_t* cache = (int32_t*)next_instr; + *cache = offset; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _Ci_specialize(next_instr, STORE_OBJ_FIELD); + _PyFrame_StackPointerInvalidate(frame); + } + } + #endif + } else { + #if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) + if (adaptive_enabled) { + if (offset <= INT32_MAX >> 8) { + assert(field_type < 0xff); + int32_t* cache = (int32_t*)next_instr; + *cache = offset << 8 | field_type; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _Ci_specialize(next_instr, STORE_PRIMITIVE_FIELD); + _PyFrame_StackPointerInvalidate(frame); + } + } + #endif + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + store_field(field_type, (char*)addr, value); + _PyFrame_StackPointerInvalidate(frame); + } + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + } else if (extop == RETURN_PRIMITIVE) { + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyStackRef temp = + sign_extend_primitive(PyStackRef_MakeHeapSafe(args[0]), extoparg); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(STACK_LEVEL() == 0); + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame* dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(frame->return_offset); + stack_pointer[0] = temp; + stack_pointer += 1; + LLTRACE_RESUME_FRAME(); + DISPATCH(); + } else if (extop == POP_JUMP_IF_ZERO) { + PyObject* cond = PyStackRef_AsPyObjectBorrow(args[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int is_nonzero = PyObject_IsTrue(cond); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + SKIP_OVER(2); + if (!is_nonzero) { + JUMPBY(extoparg); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } else if (extop == POP_JUMP_IF_NONZERO) { + PyObject* cond = PyStackRef_AsPyObjectBorrow(args[0]); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int is_nonzero = PyObject_IsTrue(cond); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + SKIP_OVER(2); + if (is_nonzero) { + JUMPBY(extoparg); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } else if (extop == CONVERT_PRIMITIVE) { + PyObject* val = PyStackRef_AsPyObjectBorrow(args[0]); + Py_ssize_t from_type = extoparg & 0xFF; + Py_ssize_t to_type = extoparg >> 4; + Py_ssize_t extend_sign = + (from_type & TYPED_INT_SIGNED) && (to_type & TYPED_INT_SIGNED); + int size = to_type >> 1; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + size_t ival = (size_t)PyLong_AsVoidPtr(val); + _PyFrame_StackPointerInvalidate(frame); + ival &= trunc_masks[size]; + if (extend_sign != 0 && (ival & signed_bits[size])) { + ival |= (signex_masks[size]); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyObject* res = PyLong_FromSize_t(ival); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + if (res == NULL) { + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + top[0] = PyStackRef_FromPyObjectSteal(res); + } else if (extop == LOAD_ITERABLE_ARG) { + PyObject* tup = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject* element; + int idx = extoparg; + _PyStackRef new_tup; + if (!PyTuple_CheckExact(tup)) { + if (tup->ob_type->tp_iter == NULL && !PySequence_Check(tup)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyErr_Format( + PyExc_TypeError, + "argument after * " + "must be an iterable, not %.200s", + tup->ob_type->tp_name); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + tup = PySequence_Tuple(tup); + _PyFrame_StackPointerInvalidate(frame); + if (tup == NULL) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + new_tup = PyStackRef_FromPyObjectSteal(tup); + } else { + new_tup = PyStackRef_FromPyObjectNew(tup); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + element = PyTuple_GetItem(tup, idx); + _PyFrame_StackPointerInvalidate(frame); + if (element == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(new_tup); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + Py_INCREF(element); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + top[0] = PyStackRef_FromPyObjectSteal(element); + top[1] = new_tup; + } else if (extop == LOAD_MAPPING_ARG) { + PyObject *defaultval, *mapping, *name; + if (extoparg == 3) { + defaultval = PyStackRef_AsPyObjectBorrow(args[0]); + mapping = PyStackRef_AsPyObjectBorrow(args[1]); + name = PyStackRef_AsPyObjectBorrow(args[2]); + } else { + defaultval = NULL; + mapping = PyStackRef_AsPyObjectBorrow(args[0]); + name = PyStackRef_AsPyObjectBorrow(args[1]); + } + PyObject* value; + if (!PyDict_Check(mapping) && !Ci_CheckedDict_Check(mapping)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyErr_Format( + PyExc_TypeError, + "argument after ** " + "must be a dict, not %.200s", + mapping->ob_type->tp_name); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + value = PyDict_GetItemWithError(mapping, name); + _PyFrame_StackPointerInvalidate(frame); + if (value == NULL) { + if (_PyErr_Occurred(tstate)) { + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } else if (oparg == 2) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyErr_Format(PyExc_TypeError, "missing argument %U", name); + _PyFrame_StackPointerInvalidate(frame); + assert(defaultval == NULL); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } else { + value = defaultval; + } + } + Py_INCREF(value); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + top[0] = PyStackRef_FromPyObjectSteal(value); + } else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyErr_Format( + PyExc_RuntimeError, "unsupported extended opcode: %d", extop); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + for (int _i = oparg >> 2; --_i >= 0;) { + PyStackRef_CLOSE(args[_i]); + } + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -(oparg & 0x03); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + stack_pointer += -(oparg & 0x03) + (oparg >> 2); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + stack_pointer += -(oparg & 0x03) + (oparg >> 2); + stack_pointer += -(oparg >> 2) + (oparg & 0x03); + } + SKIP_OVER(1); + DISPATCH(); + } + + TARGET(FORMAT_SIMPLE) { + #if _Py_TAIL_CALL_INTERP + int opcode = FORMAT_SIMPLE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_SIMPLE); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = PyObject_Format(value_o, NULL); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + else { + res = value; + stack_pointer += -1; + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(FORMAT_WITH_SPEC) { + #if _Py_TAIL_CALL_INTERP + int opcode = FORMAT_WITH_SPEC; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_WITH_SPEC); + _PyStackRef value; + _PyStackRef fmt_spec; + _PyStackRef res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = fmt_spec; + fmt_spec = PyStackRef_NULL; + stack_pointer[-1] = fmt_spec; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = value; + value = PyStackRef_NULL; + stack_pointer[-2] = value; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(FOR_ITER) { + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER); + PREDICTED_FOR_ITER:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + // _SPECIALIZE_FOR_ITER + { + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(FOR_ITER); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + // _FOR_ITER + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); + _PyFrame_StackPointerInvalidate(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + JUMP_TO_LABEL(error); + } + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + DISPATCH(); + } + next = item; + } + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(FOR_ITER_GEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_GEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_GEN); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef gen_frame; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + } + // _FOR_ITER_GEN_FRAME + { + iter = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(gen) != &PyGen_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + if (!gen_try_set_executing((PyGenObject *)gen)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + STAT_INC(FOR_ITER, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_None); + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + pushed_frame->previous = frame; + frame->return_offset = (uint16_t)( 2u + oparg); + gen_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + CI_UPDATE_CALL_COUNT + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(FOR_ITER_LIST) { + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_LIST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_LIST); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_LIST + { + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyList_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + #endif + } + // _ITER_JUMP_LIST + { + #ifdef Py_GIL_DISABLED + + #else + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + STAT_INC(FOR_ITER, hit); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + DISPATCH(); + } + #endif + } + // _ITER_NEXT_LIST + { + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyList_CheckExact(list_o)); + #ifdef Py_GIL_DISABLED + assert(_Py_IsOwnedByCurrentThread(list_o) || + _PyObject_GC_IS_SHARED(list_o)); + STAT_INC(FOR_ITER, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); + _PyFrame_StackPointerInvalidate(frame); + if (result < 0) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + if (result == 0) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + DISPATCH(); + } + #else + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); + #endif + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + } + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(FOR_ITER_RANGE) { + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_RANGE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_RANGE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_RANGE + { + iter = stack_pointer[-2]; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + #endif + } + // _ITER_JUMP_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); + #endif + STAT_INC(FOR_ITER, hit); + if (r->len <= 0) { + JUMPBY(oparg + 1); + DISPATCH(); + } + } + // _ITER_NEXT_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); + #endif + assert(r->len > 0); + long value = r->start; + r->start = value + r->step; + r->len--; + PyObject *res = PyLong_FromLong(value); + if (res == NULL) { + JUMP_TO_LABEL(error); + } + next = PyStackRef_FromPyObjectSteal(res); + } + stack_pointer[0] = next; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(FOR_ITER_TUPLE) { + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_TUPLE + { + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyTuple_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + } + // _ITER_JUMP_TUPLE + { + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + (void)tuple_o; + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + STAT_INC(FOR_ITER, hit); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + DISPATCH(); + } + } + // _ITER_NEXT_TUPLE + { + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + uintptr_t i = PyStackRef_UntagInt(null_or_index); + assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + } + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(FOR_ITER_VIRTUAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_VIRTUAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_VIRTUAL); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _GUARD_TOS_NOT_NULL + { + null_or_index = stack_pointer[-1]; + if (PyStackRef_IsNull(null_or_index)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + } + // _FOR_ITER_VIRTUAL + { + iter = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + Py_ssize_t index = PyStackRef_UntagInt(null_or_index); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index); + _PyFrame_StackPointerInvalidate(frame); + PyObject *next_o = next_index.object; + index = next_index.index; + if (next_o == NULL) { + if (index < 0) { + JUMP_TO_LABEL(error); + } + JUMPBY(oparg + 1); + DISPATCH(); + } + null_or_index = PyStackRef_TagInt(index); + next = PyStackRef_FromPyObjectSteal(next_o); + } + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(GET_AITER) { + #if _Py_TAIL_CALL_INTERP + int opcode = GET_AITER; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AITER); + _PyStackRef obj; + _PyStackRef iter; + obj = stack_pointer[-1]; + unaryfunc getter = NULL; + PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); + PyObject *iter_o; + PyTypeObject *type = Py_TYPE(obj_o); + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; + } + if (getter == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an object with " + "__aiter__ method, got %.100s", + type->tp_name); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(obj); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + iter_o = (*getter)(obj_o); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(obj); + _PyFrame_StackPointerInvalidate(frame); + if (iter_o == NULL) { + JUMP_TO_LABEL(error); + } + if (Py_TYPE(iter_o)->tp_as_async == NULL || + Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' received an object from __aiter__ " + "that does not implement __anext__: %.100s", + Py_TYPE(iter_o)->tp_name); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(iter_o); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[0] = iter; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(GET_ANEXT) { + #if _Py_TAIL_CALL_INTERP + int opcode = GET_ANEXT; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ANEXT); + _PyStackRef aiter; + _PyStackRef awaitable; + aiter = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); + _PyFrame_StackPointerInvalidate(frame); + if (awaitable_o == NULL) { + JUMP_TO_LABEL(error); + } + awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); + stack_pointer[0] = awaitable; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(GET_AWAITABLE) { + #if _Py_TAIL_CALL_INTERP + int opcode = GET_AWAITABLE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AWAITABLE); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(iterable); + _PyFrame_StackPointerInvalidate(frame); + if (iter_o == NULL) { + JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[0] = iter; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(GET_ITER) { + #if _Py_TAIL_CALL_INTERP + int opcode = GET_ITER; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(GET_ITER); + PREDICTED_GET_ITER:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef iterable; + _PyStackRef iter; + _PyStackRef index_or_null; + // _SPECIALIZE_GET_ITER + { + iterable = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_GetIter(iterable, next_instr); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(GET_ITER); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + // _GET_ITER + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyStackRef result = _PyEval_GetIter(iterable, &index_or_null, oparg); + _PyFrame_StackPointerInvalidate(frame); + if (PyStackRef_IsError(result)) { + JUMP_TO_LABEL(pop_1_error); + } + iter = result; + } + stack_pointer[-1] = iter; + stack_pointer[0] = index_or_null; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(GET_ITER_SELF) { + #if _Py_TAIL_CALL_INTERP + int opcode = GET_ITER_SELF; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(GET_ITER_SELF); + static_assert(INLINE_CACHE_ENTRIES_GET_ITER == 1, "incorrect cache size"); + _PyStackRef iterable; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_ITERATOR + { + iterable = stack_pointer[-1]; + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable)); + if (tp->tp_iter != PyObject_SelfIter) { + UPDATE_MISS_STATS(GET_ITER); + assert(_PyOpcode_Deopt[opcode] == (GET_ITER)); + JUMP_TO_PREDICTED(GET_ITER); + } + STAT_INC(GET_ITER, hit); + } + // _PUSH_NULL + { + res = PyStackRef_NULL; + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(GET_ITER_VIRTUAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = GET_ITER_VIRTUAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(GET_ITER_VIRTUAL); + static_assert(INLINE_CACHE_ENTRIES_GET_ITER == 1, "incorrect cache size"); + _PyStackRef iterable; + _PyStackRef zero; + /* Skip 1 cache entry */ + // _GUARD_ITER_VIRTUAL + { + iterable = stack_pointer[-1]; + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable)); + if (tp->_tp_iteritem == NULL) { + UPDATE_MISS_STATS(GET_ITER); + assert(_PyOpcode_Deopt[opcode] == (GET_ITER)); + JUMP_TO_PREDICTED(GET_ITER); + } + STAT_INC(GET_ITER, hit); + } + // _PUSH_TAGGED_ZERO + { + zero = PyStackRef_TagInt(0); + } + stack_pointer[0] = zero; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(GET_LEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = GET_LEN; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_LEN); + _PyStackRef obj; + _PyStackRef len; + obj = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + _PyFrame_StackPointerInvalidate(frame); + if (len_i < 0) { + JUMP_TO_LABEL(error); + } + PyObject *len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) { + JUMP_TO_LABEL(error); + } + len = PyStackRef_FromPyObjectSteal(len_o); + stack_pointer[0] = len; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(IMPORT_FROM) { + #if _Py_TAIL_CALL_INTERP + int opcode = IMPORT_FROM; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_FROM); + _PyStackRef from; + _PyStackRef res; + from = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *res_o; + if (PyLazyImport_CheckExact(PyStackRef_AsPyObjectBorrow(from))) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + res_o = _PyEval_LazyImportFrom( + tstate, frame, PyStackRef_AsPyObjectBorrow(from), name); + _PyFrame_StackPointerInvalidate(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + res_o = _PyEval_ImportFrom( + tstate, PyStackRef_AsPyObjectBorrow(from), name); + _PyFrame_StackPointerInvalidate(frame); + } + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(IMPORT_NAME) { + #if _Py_TAIL_CALL_INTERP + int opcode = IMPORT_NAME; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_NAME); + _PyStackRef level; + _PyStackRef fromlist; + _PyStackRef res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyObject *res_o; + if (!(oparg & 0x02)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + res_o = _PyEval_LazyImportName(tstate, BUILTINS(), GLOBALS(), + LOCALS(), name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level), + oparg & 0x01); + _PyFrame_StackPointerInvalidate(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + res_o = _PyEval_ImportName(tstate, BUILTINS(), GLOBALS(), + LOCALS(), name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + _PyFrame_StackPointerInvalidate(frame); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = fromlist; + fromlist = PyStackRef_NULL; + stack_pointer[-1] = fromlist; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = level; + level = PyStackRef_NULL; + stack_pointer[-2] = level; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(INSTRUMENTED_CALL) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_CALL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL); + opcode = INSTRUMENTED_CALL; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef func; + _PyStackRef maybe_self; + _PyStackRef *args; + _PyStackRef res; + /* Skip 3 cache entries */ + // _MAYBE_EXPAND_METHOD + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + } + // _MONITOR_CALL + { + args = &stack_pointer[-oparg]; + maybe_self = self_or_null; + func = callable; + int is_meth = !PyStackRef_IsNull(maybe_self); + PyObject *function = PyStackRef_AsPyObjectBorrow(func); + PyObject *arg0; + if (is_meth) { + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); + } + else if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg0 = &_PyInstrumentation_MISSING; + } + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = maybe_self; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg0 + ); + _PyFrame_StackPointerInvalidate(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + // _DO_CALL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (Py_TYPE(callable_o) == &PyFunction_Type && + !IS_PEP523_HOOKED(tstate) && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + frame->return_offset = 4u ; + DISPATCH_INLINED(new_frame); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( + callable, + arguments, + total_args, + PyStackRef_NULL, + opcode == INSTRUMENTED_CALL, + frame, + this_instr, + tstate); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics_at_end(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_CALL_FUNCTION_EX; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); + opcode = INSTRUMENTED_CALL_FUNCTION_EX; + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef func_st; + _PyStackRef null; + _PyStackRef callargs_st; + _PyStackRef kwargs_st; + _PyStackRef result; + /* Skip 1 cache entry */ + // _MAKE_CALLARGS_A_TUPLE + { + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (!PyTuple_CheckExact(callargs_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + _PyFrame_StackPointerInvalidate(frame); + if (tuple_o == NULL) { + JUMP_TO_LABEL(error); + } + _PyStackRef temp = callargs; + callargs = PyStackRef_FromPyObjectSteal(tuple_o); + stack_pointer[-2] = callargs; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + } + // _DO_CALL_FUNCTION_EX + { + kwargs_st = stack_pointer[-1]; + callargs_st = callargs; + null = stack_pointer[-3]; + func_st = func; + (void)null; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + _PyFrame_StackPointerInvalidate(frame); + if (err) { + JUMP_TO_LABEL(error); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + result_o = PyObject_Call(func, callargs, kwargs); + _PyFrame_StackPointerInvalidate(frame); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result_o == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + _PyFrame_StackPointerInvalidate(frame); + } + else { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_CLEAR(result_o); + _PyFrame_StackPointerInvalidate(frame); + } + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + !IS_PEP523_HOOKED(tstate) && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + assert( 2u == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX); + frame->return_offset = 2u ; + DISPATCH_INLINED(new_frame); + } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + result_o = PyObject_Call(func, callargs, kwargs); + _PyFrame_StackPointerInvalidate(frame); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(kwargs_st); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(callargs_st); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(func_st); + _PyFrame_StackPointerInvalidate(frame); + if (result_o == NULL) { + JUMP_TO_LABEL(error); + } + result = PyStackRef_FromPyObjectSteal(result_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = result; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics_at_end(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(INSTRUMENTED_CALL_KW) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_CALL_KW; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + opcode = INSTRUMENTED_CALL_KW; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + } + // _MONITOR_CALL_KW + { + args = &stack_pointer[-1 - oparg]; + int is_meth = !PyStackRef_IsNull(self_or_null); + PyObject *arg; + if (is_meth) { + arg = PyStackRef_AsPyObjectBorrow(self_or_null); + } + else if (args) { + arg = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg = &_PyInstrumentation_MISSING; + } + PyObject *function = PyStackRef_AsPyObjectBorrow(callable); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + _PyFrame_StackPointerInvalidate(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + // _DO_CALL_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + if (Py_TYPE(callable_o) == &PyFunction_Type && + !IS_PEP523_HOOKED(tstate) && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4u ; + DISPATCH_INLINED(new_frame); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( + callable, + arguments, + total_args, + kwnames, + opcode == INSTRUMENTED_CALL_KW, + frame, + this_instr, + tstate); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(INSTRUMENTED_END_ASYNC_FOR) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_END_ASYNC_FOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_ASYNC_FOR); + _PyStackRef awaitable_st; + _PyStackRef exc_st; + // _MONITOR_END_ASYNC_FOR + { + assert((next_instr-oparg)->op.code == END_SEND || (next_instr-oparg)->op.code >= MIN_INSTRUMENTED_OPCODE); + INSTRUMENTED_JUMP(next_instr-oparg, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + // _END_ASYNC_FOR + { + exc_st = stack_pointer[-1]; + awaitable_st = stack_pointer[-2]; + JUMPBY(0); + (void)oparg; + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(exc && PyExceptionInstance_Check(exc)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); + _PyFrame_StackPointerInvalidate(frame); + if (matches) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = exc_st; + exc_st = PyStackRef_NULL; + stack_pointer[-1] = exc_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = awaitable_st; + awaitable_st = PyStackRef_NULL; + stack_pointer[-2] = awaitable_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + } + else { + Py_INCREF(exc); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_SetRaisedException(tstate, exc); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + JUMP_TO_LABEL(exception_unwind); + } + } + DISPATCH(); + } + + TARGET(INSTRUMENTED_END_FOR) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_END_FOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_FOR); + _PyStackRef receiver; + _PyStackRef value; + value = stack_pointer[-1]; + receiver = stack_pointer[-3]; + if (PyStackRef_GenCheck(receiver)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + _PyFrame_StackPointerInvalidate(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH(); + } + + TARGET(INSTRUMENTED_END_SEND) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_END_SEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_SEND); + _PyStackRef receiver; + _PyStackRef index_or_null; + _PyStackRef value; + _PyStackRef val; + value = stack_pointer[-1]; + index_or_null = stack_pointer[-2]; + receiver = stack_pointer[-3]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + _PyFrame_StackPointerInvalidate(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + val = value; + (void)index_or_null; + stack_pointer[-3] = val; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(receiver); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH(); + } + + TARGET(INSTRUMENTED_FOR_ITER) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_FOR_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + /* Skip 1 cache entry */ + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); + _PyFrame_StackPointerInvalidate(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + JUMP_TO_LABEL(error); + } + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + DISPATCH(); + } + next = item; + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(INSTRUMENTED_INSTRUCTION) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_INSTRUCTION; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + opcode = INSTRUMENTED_INSTRUCTION; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + if (next_opcode < 0) { + JUMP_TO_LABEL(error); + } + next_instr = this_instr; + if (_PyOpcode_Caches[next_opcode]) { + PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); + } + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); + } + + TARGET(INSTRUMENTED_JUMP_BACKWARD) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_JUMP_BACKWARD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics(tstate); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + // _MONITOR_JUMP_BACKWARD + { + INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); + } + DISPATCH(); + } + + TARGET(INSTRUMENTED_JUMP_FORWARD) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_JUMP_FORWARD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); + DISPATCH(); + } + + TARGET(INSTRUMENTED_LINE) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_LINE; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_LINE); + opcode = INSTRUMENTED_LINE; + int original_opcode = 0; + if (tstate->tracing) { + PyCodeObject *code = _PyFrame_GetCode(frame); + int index = (int)(this_instr - _PyFrame_GetBytecode(frame)); + original_opcode = code->_co_monitoring->lines->data[index*code->_co_monitoring->lines->bytes_per_entry]; + next_instr = this_instr; + } else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + original_opcode = _Py_call_instrumentation_line( + tstate, frame, this_instr, prev_instr); + _PyFrame_StackPointerInvalidate(frame); + // Explicit stack reload + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackAssertInvalid(frame); + if (original_opcode < 0) { + next_instr = this_instr+1; + JUMP_TO_LABEL(error); + } + next_instr = frame->instr_ptr; + if (next_instr != this_instr) { + DISPATCH(); + } + } + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + PAUSE_ADAPTIVE_COUNTER(cache->counter); + } + opcode = original_opcode; + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); + } + + TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); + opcode = INSTRUMENTED_LOAD_SUPER_ATTR; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + _PyFrame_StackPointerInvalidate(frame); + if (err) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + } + PyObject *super; + { + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + _PyFrame_StackPointerInvalidate(frame); + } + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + _PyFrame_StackPointerInvalidate(frame); + } + else { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_CLEAR(super); + _PyFrame_StackPointerInvalidate(frame); + } + } + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (super == NULL) { + JUMP_TO_LABEL(error); + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *attr_o = PyObject_GetAttr(super, name); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(super); + _PyFrame_StackPointerInvalidate(frame); + if (attr_o == NULL) { + JUMP_TO_LABEL(error); + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = attr; + stack_pointer += 1 + (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(INSTRUMENTED_NOT_TAKEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_NOT_TAKEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); + (void)this_instr; + INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_ITER) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); + _PyStackRef iter; + _PyStackRef index_or_null; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; + INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(iter); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); + _PyStackRef value; + /* Skip 1 cache entry */ + value = stack_pointer[-1]; + int jump = PyStackRef_IsNone(value); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + else { + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += 1; + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); + _PyStackRef value; + /* Skip 1 cache entry */ + value = stack_pointer[-1]; + int jump = !PyStackRef_IsNone(value); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + else { + stack_pointer += -1; + } + DISPATCH(); + } + + TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(INSTRUMENTED_RESUME) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_RESUME; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_RESUME); + /* Skip 1 cache entry */ + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + _PyFrame_StackPointerInvalidate(frame); + if (bytecode == NULL) { + JUMP_TO_LABEL(error); + } + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + next_instr = frame->instr_ptr; + DISPATCH(); + } + #endif + } + // _MAYBE_INSTRUMENT + { + #ifdef Py_GIL_DISABLED + + int check_instrumentation = 1; + #else + int check_instrumentation = (tstate->tracing == 0); + #endif + if (check_instrumentation) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + _PyFrame_StackPointerInvalidate(frame); + if (err) { + JUMP_TO_LABEL(error); + } + next_instr = this_instr; + DISPATCH(); + } + } + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics(tstate); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + } + // _MONITOR_RESUME + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _Py_call_instrumentation( + tstate, oparg == 0 ? PY_MONITORING_EVENT_PY_START : PY_MONITORING_EVENT_PY_RESUME, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + // Explicit stack reload + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackAssertInvalid(frame); + if (err) { + JUMP_TO_LABEL(error); + } + if (frame->instr_ptr != this_instr) { + next_instr = frame->instr_ptr; + } + } + DISPATCH(); + } + + TARGET(INSTRUMENTED_RETURN_VALUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_RETURN_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); + _PyStackRef val; + _PyStackRef value; + _PyStackRef retval; + _PyStackRef res; + // _RETURN_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + _PyFrame_StackPointerInvalidate(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + // _MAKE_HEAP_SAFE + { + value = val; + value = PyStackRef_MakeHeapSafe(value); + } + // _RETURN_VALUE + { + retval = value; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyStackRef temp = retval; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(STACK_LEVEL() == 0); + DTRACE_FUNCTION_RETURN(); + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(frame->return_offset); + res = temp; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(INSTRUMENTED_YIELD_VALUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_YIELD_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + opcode = INSTRUMENTED_YIELD_VALUE; + _PyStackRef val; + _PyStackRef value; + _PyStackRef retval; + // _YIELD_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + _PyFrame_StackPointerInvalidate(frame); + if (err) { + // Explicit stack reload + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackAssertInvalid(frame); + JUMP_TO_LABEL(error); + } + if (frame->instr_ptr != this_instr) { + // Explicit stack reload + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackAssertInvalid(frame); + next_instr = frame->instr_ptr; + DISPATCH(); + } + } + // _MAKE_HEAP_SAFE + { + value = val; + value = PyStackRef_MakeHeapSafe(value); + } + // _YIELD_VALUE + { + retval = value; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + _PyStackRef temp = retval; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + DTRACE_FUNCTION_RETURN(); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; + FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE && defined(Py_DEBUG) + if (!PyStackRef_IsNone(frame->f_executable)) { + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; + assert(opcode == SEND || opcode == FOR_ITER); + } + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + value = temp; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(INTERPRETER_EXIT) { + #if _Py_TAIL_CALL_INTERP + int opcode = INTERPRETER_EXIT; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INTERPRETER_EXIT); + _PyStackRef retval; + retval = stack_pointer[-1]; + assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); + assert(_PyFrame_IsIncomplete(frame)); + tstate->current_frame = frame->previous; + assert(!_PyErr_Occurred(tstate)); + PyObject *result = PyStackRef_AsPyObjectSteal(retval); + #if !_Py_TAIL_CALL_INTERP + assert(frame == &entry.frame); + #endif + #ifdef _Py_TIER2 + _PyStackRef executor = frame->localsplus[0]; + assert(tstate->current_executor == NULL); + if (!PyStackRef_IsNull(executor)) { + assert(PyStackRef_TYPE(executor) == &_PyUOpExecutor_Type); + tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(executor); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += 1; + } + #endif + LLTRACE_RESUME_FRAME(); + return result; + } + + TARGET(IS_OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = IS_OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IS_OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + _PyStackRef l; + _PyStackRef r; + _PyStackRef value; + // _IS_OP + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + b = res ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + } + // _POP_TOP + { + value = r; + stack_pointer[-2] = b; + stack_pointer[-1] = l; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = l; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(JUMP_BACKWARD) { + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD); + PREDICTED_JUMP_BACKWARD:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + /* Skip 1 cache entry */ + // _SPECIALIZE_JUMP_BACKWARD + { + #if ENABLE_SPECIALIZATION + if (this_instr->op.code == JUMP_BACKWARD) { + uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); + next_instr = this_instr; + DISPATCH_SAME_OPARG(); + } + #endif + } + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics(tstate); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } + DISPATCH(); + } + + TARGET(JUMP_BACKWARD_JIT) { + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD_JIT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD_JIT); + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics(tstate); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } + // _JIT + { + #ifdef _Py_TIER2 + bool is_resume = this_instr->op.code == RESUME_CHECK_JIT; + _Py_BackoffCounter counter = this_instr[1].counter; + if ((backoff_counter_triggers(counter) && + !IS_JIT_TRACING() && + (this_instr->op.code == JUMP_BACKWARD_JIT || is_resume)) && + next_instr->op.code != ENTER_EXECUTOR) { + _Py_CODEUNIT *insert_exec_at = this_instr; + for (int tmp = oparg; tmp > 255; tmp >>= 8) { + insert_exec_at--; + } + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, + is_resume ? insert_exec_at : next_instr, stack_pointer, 0, NULL, oparg, NULL); + if (succ) { + ENTER_TRACING(); + } + else { + this_instr[1].counter = restart_backoff_counter(counter); + } + } + else { + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + #endif + } + DISPATCH(); + } + + TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD_NO_INTERRUPT; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + DISPATCH(); + } + + TARGET(JUMP_BACKWARD_NO_JIT) { + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD_NO_JIT; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics(tstate); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } + DISPATCH(); + } + + TARGET(JUMP_FORWARD) { + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_FORWARD; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_FORWARD); + JUMPBY(oparg); + DISPATCH(); + } + + TARGET(LIST_APPEND) { + #if _Py_TAIL_CALL_INTERP + int opcode = LIST_APPEND; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_APPEND); + _PyStackRef list; + _PyStackRef v; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg - 1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = Ci_ListOrCheckedList_Append( + (PyListObject*)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(v); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH(); + } + + TARGET(LIST_EXTEND) { + #if _Py_TAIL_CALL_INTERP + int opcode = LIST_EXTEND; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_EXTEND); + _PyStackRef list_st; + _PyStackRef iterable_st; + _PyStackRef i; + _PyStackRef value; + // _LIST_EXTEND + { + iterable_st = stack_pointer[-1]; + list_st = stack_pointer[-2 - (oparg-1)]; + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + _PyFrame_StackPointerInvalidate(frame); + if (none_val == NULL) { + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + if (matches && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_Clear(tstate); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_Format(tstate, PyExc_TypeError, + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); + _PyFrame_StackPointerInvalidate(frame); + } + JUMP_TO_LABEL(error); + } + assert(Py_IsNone(none_val)); + i = iterable_st; + } + // _POP_TOP + { + value = i; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(LOAD_ATTR) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR); + PREDICTED_LOAD_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 10; + (void)this_instr; + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *self_or_null; + // _SPECIALIZE_LOAD_ATTR + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_LoadAttr(owner, next_instr, name); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + /* Skip 8 cache entries */ + // _LOAD_ATTR + { + self_or_null = &stack_pointer[0]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + if (oparg & 1) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + attr = _Py_LoadAttr_StackRefSteal(tstate, owner, name, self_or_null); + _PyFrame_StackPointerInvalidate(frame); + if (PyStackRef_IsNull(attr)) { + JUMP_TO_LABEL(pop_1_error); + } + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + attr = _PyObject_GetAttrStackRef(PyStackRef_AsPyObjectBorrow(owner), name); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer[-1] = attr; + stack_pointer += (oparg&1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(owner); + _PyFrame_StackPointerInvalidate(frame); + if (PyStackRef_IsNull(attr)) { + JUMP_TO_LABEL(error); + } + stack_pointer += -(oparg&1); + } + } + stack_pointer[-1] = attr; + stack_pointer += (oparg&1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_CLASS) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_CLASS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _CHECK_ATTR_CLASS + { + uint32_t type_version = read_u32(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME + { + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *getattribute = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)getattribute; + assert(func_version != 0); + if (f->func_version != func_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 2); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + STAT_INC(LOAD_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked( + tstate, PyStackRef_FromPyObjectNew(f), 2, frame); + pushed_frame->localsplus[0] = owner; + pushed_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + CI_UPDATE_CALL_COUNT + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(LOAD_ATTR_INSTANCE_VALUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_INSTANCE_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef o; + _PyStackRef value; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _CHECK_MANAGED_OBJECT_HAS_VALUES + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _LOAD_ATTR_INSTANCE_VALUE + { + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr); + if (!increfed) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + o = owner; + } + // _POP_TOP + { + value = o; + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_METHOD_LAZY_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _CHECK_ATTR_METHOD_LAZY_DICT + { + uint16_t dictoffset = read_u16(&this_instr[4].cache); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); + if (dict != NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + /* Skip 1 cache entry */ + // _LOAD_ATTR_METHOD_LAZY_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_METHOD_NO_DICT) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_METHOD_NO_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_METHOD_WITH_VALUES; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _CHECK_MANAGED_OBJECT_HAS_VALUES + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_MODULE) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_MODULE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef o; + _PyStackRef value; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _LOAD_ATTR_MODULE + { + owner = stack_pointer[-1]; + uint32_t dict_version = read_u32(&this_instr[2].cache); + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; + assert(dict != NULL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + assert(keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_CONSUME(ep->me_value); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + o = owner; + } + // _POP_TOP + { + value = o; + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(owner); + _PyFrame_StackPointerInvalidate(frame); + attr = PyStackRef_FromPyObjectNew(descr); + } + stack_pointer[0] = attr; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _CHECK_MANAGED_OBJECT_HAS_VALUES + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(owner); + _PyFrame_StackPointerInvalidate(frame); + attr = PyStackRef_FromPyObjectNew(descr); + } + stack_pointer[0] = attr; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_PROPERTY) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_PROPERTY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _LOAD_ATTR_PROPERTY_FRAME + { + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *fget = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + if (f->func_version != func_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + PyCodeObject *code = (PyCodeObject *)f->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + STAT_INC(LOAD_ATTR, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); + pushed_frame->localsplus[0] = owner; + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + CI_UPDATE_CALL_COUNT + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(LOAD_ATTR_SLOT) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_SLOT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef o; + _PyStackRef value; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _LOAD_ATTR_SLOT + { + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + o = owner; + } + // _POP_TOP + { + value = o; + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_ATTR_WITH_HINT) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_WITH_HINT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef o; + _PyStackRef value; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _LOAD_ATTR_WITH_HINT + { + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (dict == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + PyDictKeysObject *dk = FT_ATOMIC_LOAD_PTR(dict->ma_keys); + assert(PyDict_CheckExact((PyObject *)dict)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread((PyObject *)dict) && !_PyObject_GC_IS_SHARED(dict)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + #endif + PyObject *attr_o; + if (hint >= (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_nentries)) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (dk->dk_kind != DICT_KEYS_UNICODE) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dk) + hint; + if (FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key) != name) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + attr_o = FT_ATOMIC_LOAD_PTR(ep->me_value); + if (attr_o == NULL) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + STAT_INC(LOAD_ATTR, hit); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + o = owner; + } + // _POP_TOP + { + value = o; + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_BUILD_CLASS) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_BUILD_CLASS; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_BUILD_CLASS); + _PyStackRef bc; + int err; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *bc_o = _PyMapping_GetOptionalItem2(BUILTINS(), &_Py_ID(__build_class__), &err); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (bc_o == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + bc = PyStackRef_FromPyObjectSteal(bc_o); + stack_pointer[0] = bc; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_COMMON_CONSTANT) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_COMMON_CONSTANT; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); + _PyStackRef value; + assert(oparg < NUM_COMMON_CONSTANTS); + value = PyStackRef_FromPyObjectNew(Ci_common_consts[oparg]); + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_CONST) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_CONST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST); + _PyStackRef value; + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectBorrow(obj); + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_DEREF) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_DEREF; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_DEREF); + _PyStackRef value; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + value = _PyCell_GetStackRef(cell); + _PyFrame_StackPointerInvalidate(frame); + if (PyStackRef_IsNull(value)) { + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST); + _PyStackRef value; + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_FAST_AND_CLEAR) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_AND_CLEAR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); + _PyStackRef value; + value = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_FAST_BORROW) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_BORROW; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_BORROW); + _PyStackRef value; + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_FAST_BORROW_LOAD_FAST_BORROW) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_BORROW_LOAD_FAST_BORROW); + _PyStackRef value1; + _PyStackRef value2; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = PyStackRef_Borrow(GETLOCAL(oparg1)); + value2 = PyStackRef_Borrow(GETLOCAL(oparg2)); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_FAST_CHECK) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_CHECK; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_CHECK); + _PyStackRef value; + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + value = PyStackRef_DUP(value_s); + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_FAST_LOAD_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_LOAD_FAST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); + _PyStackRef value1; + _PyStackRef value2; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = PyStackRef_DUP(GETLOCAL(oparg1)); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_FROM_DICT_OR_DEREF) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FROM_DICT_OR_DEREF; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); + _PyStackRef class_dict_st; + _PyStackRef value; + class_dict_st = stack_pointer[-1]; + PyObject *name; + PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); + assert(class_dict); + assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + int err; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject* value_o = _PyMapping_GetOptionalItem2(class_dict, name, &err); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (!value_o) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(class_dict_st); + _PyFrame_StackPointerInvalidate(frame); + value = PyStackRef_FromPyObjectSteal(value_o); + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_FROM_DICT_OR_GLOBALS) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FROM_DICT_OR_GLOBALS; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); + _PyStackRef mod_or_class_dict; + _PyStackRef v; + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *v_o = _PyMapping_GetOptionalItem2(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &err); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(mod_or_class_dict); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (v_o == NULL) { + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + _PyFrame_StackPointerInvalidate(frame); + if (v_o == NULL) { + if (!_PyErr_Occurred(tstate)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + _PyFrame_StackPointerInvalidate(frame); + } + JUMP_TO_LABEL(error); + } + if (PyLazyImport_CheckExact(v_o)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_SETREF(v_o, l_v); + _PyFrame_StackPointerInvalidate(frame); + if (v_o == NULL) { + JUMP_TO_LABEL(error); + } + } + } + else { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + v_o = _PyMapping_GetOptionalItem2(GLOBALS(), name, &err); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (v_o == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + v_o = _PyMapping_GetOptionalItem2(BUILTINS(), name, &err); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (v_o == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + } + if (PyLazyImport_CheckExact(v_o)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_SETREF(v_o, l_v); + _PyFrame_StackPointerInvalidate(frame); + if (v_o == NULL) { + JUMP_TO_LABEL(error); + } + } + } + } + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[0] = v; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_GLOBAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_GLOBAL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL); + PREDICTED_LOAD_GLOBAL:; + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + _PyStackRef *res; + _PyStackRef *null; + // _SPECIALIZE_LOAD_GLOBAL + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_GLOBAL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + // _LOAD_GLOBAL + { + res = &stack_pointer[0]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); + _PyFrame_StackPointerInvalidate(frame); + if (PyStackRef_IsNull(*res)) { + JUMP_TO_LABEL(error); + } + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += 1 + (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_GLOBAL_BUILTIN) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_GLOBAL_BUILTIN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + _PyStackRef res; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + assert(keys->dk_kind == DICT_KEYS_UNICODE); + } + // _LOAD_GLOBAL_BUILTINS + { + uint16_t version = read_u16(&this_instr[3].cache); + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + assert(keys->dk_kind == DICT_KEYS_UNICODE); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_CONSUME(entries[index].me_value); + if (res_o == NULL) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + #else + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = res; + stack_pointer += 1 + (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_GLOBAL_MODULE) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_GLOBAL_MODULE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + _PyStackRef res; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _NOP + { + } + // _LOAD_GLOBAL_MODULE + { + uint16_t version = read_u16(&this_instr[2].cache); + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + assert(keys->dk_kind == DICT_KEYS_UNICODE); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); + assert(index < DK_SIZE(keys)); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_CONSUME(entries[index].me_value); + if (res_o == NULL) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + #else + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = res; + stack_pointer += 1 + (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_LOCALS) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_LOCALS; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_LOCALS); + _PyStackRef locals; + PyObject *l = LOCALS(); + if (l == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + locals = PyStackRef_FromPyObjectNew(l); + stack_pointer[0] = locals; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_NAME) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_NAME; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_NAME); + _PyStackRef v; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + _PyFrame_StackPointerInvalidate(frame); + if (v_o == NULL) { + JUMP_TO_LABEL(error); + } + if (PyLazyImport_CheckExact(v_o)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); + _PyFrame_StackPointerInvalidate(frame); + if (l_v == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(v_o); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = PyDict_SetItem(GLOBALS(), name, l_v); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(v_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(l_v); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_SETREF(v_o, l_v); + _PyFrame_StackPointerInvalidate(frame); + } + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[0] = v; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_SMALL_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SMALL_INT; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SMALL_INT); + _PyStackRef value; + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_SPECIAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SPECIAL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SPECIAL); + _PyStackRef self; + _PyStackRef *method_and_self; + // _INSERT_NULL + { + self = stack_pointer[-1]; + method_and_self = &stack_pointer[-1]; + method_and_self[1] = self; + method_and_self[0] = PyStackRef_NULL; + } + // _LOAD_SPECIAL + { + method_and_self = &stack_pointer[-1]; + PyObject *name = _Py_SpecialMethods[oparg].name; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _PyObject_LookupSpecialMethod(name, method_and_self); + _PyFrame_StackPointerInvalidate(frame); + if (err <= 0) { + if (err == 0) { + PyObject *owner = PyStackRef_AsPyObjectBorrow(method_and_self[1]); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + const char *errfmt = _PyEval_SpecialMethodCanSuggest(owner, oparg) + ? _Py_SpecialMethods[oparg].error_suggestion + : _Py_SpecialMethods[oparg].error; + _PyFrame_StackPointerInvalidate(frame); + assert(!_PyErr_Occurred(tstate)); + assert(errfmt != NULL); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_Format(tstate, PyExc_TypeError, errfmt, owner); + _PyFrame_StackPointerInvalidate(frame); + } + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(LOAD_SUPER_ATTR) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SUPER_ATTR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR); + PREDICTED_LOAD_SUPER_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = LOAD_SUPER_ATTR; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef *null; + // _SPECIALIZE_LOAD_SUPER_ATTR + { + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + _PyFrame_StackPointerInvalidate(frame); + if (err) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + } + PyObject *super; + { + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + _PyFrame_StackPointerInvalidate(frame); + } + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + _PyFrame_StackPointerInvalidate(frame); + } + else { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_CLEAR(super); + _PyFrame_StackPointerInvalidate(frame); + } + } + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (super == NULL) { + JUMP_TO_LABEL(error); + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *attr_o = PyObject_GetAttr(super, name); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(super); + _PyFrame_StackPointerInvalidate(frame); + if (attr_o == NULL) { + JUMP_TO_LABEL(error); + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = attr; + stack_pointer += 1 + (oparg & 1); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_SUPER_ATTR_ATTR) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SUPER_ATTR_ATTR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr_st; + /* Skip 1 cache entry */ + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(!(oparg & 1)); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (attr == NULL) { + JUMP_TO_LABEL(error); + } + attr_st = PyStackRef_FromPyObjectSteal(attr); + stack_pointer[0] = attr_st; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(LOAD_SUPER_ATTR_METHOD) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SUPER_ATTR_METHOD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef self_or_null; + /* Skip 1 cache entry */ + // _GUARD_LOAD_SUPER_ATTR_METHOD + { + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + assert(oparg & 1); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } + } + // _LOAD_SUPER_ATTR_METHOD + { + self_st = stack_pointer[-1]; + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyTypeObject *cls = (PyTypeObject *)class; + int method_found = 0; + PyObject *attr_o; + { + int *method_found_ptr = &method_found; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); + _PyFrame_StackPointerInvalidate(frame); + } + if (attr_o == NULL) { + JUMP_TO_LABEL(error); + } + if (method_found) { + self_or_null = self_st; + } else { + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(self_st); + _PyFrame_StackPointerInvalidate(frame); + self_or_null = PyStackRef_NULL; + stack_pointer += 1; + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = global_super_st; + global_super_st = self_or_null; + stack_pointer[-2] = global_super_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-1] = class_st; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + stack_pointer[0] = attr; + stack_pointer[1] = self_or_null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(MAKE_CELL) { + #if _Py_TAIL_CALL_INTERP + int opcode = MAKE_CELL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_CELL); + PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { + JUMP_TO_LABEL(error); + } + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH(); + } + + TARGET(MAKE_FUNCTION) { + #if _Py_TAIL_CALL_INTERP + int opcode = MAKE_FUNCTION; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_FUNCTION); + _PyStackRef codeobj_st; + _PyStackRef func; + _PyStackRef co; + _PyStackRef value; + // _MAKE_FUNCTION + { + codeobj_st = stack_pointer[-1]; + PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + _PyFrame_StackPointerInvalidate(frame); + if (func_obj == NULL) { + JUMP_TO_LABEL(error); + } + co = codeobj_st; + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); + func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); + } + // _POP_TOP + { + value = co; + stack_pointer[-1] = func; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(MAP_ADD) { + #if _Py_TAIL_CALL_INTERP + int opcode = MAP_ADD; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAP_ADD); + _PyStackRef dict_st; + _PyStackRef key; + _PyStackRef value; + value = stack_pointer[-1]; + key = stack_pointer[-2]; + dict_st = stack_pointer[-3 - (oparg - 1)]; + PyObject* dict = PyStackRef_AsPyObjectBorrow(dict_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = Ci_DictOrChecked_SetItem( + dict, + PyStackRef_AsPyObjectBorrow(key), + PyStackRef_AsPyObjectBorrow(value)); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(key); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + DISPATCH(); + } + + TARGET(MATCH_CLASS) { + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_CLASS; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_CLASS); + _PyStackRef subject; + _PyStackRef type; + _PyStackRef names; + _PyStackRef attrs; + _PyStackRef s; + _PyStackRef tp; + _PyStackRef n; + _PyStackRef value; + // _MATCH_CLASS + { + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); + _PyFrame_StackPointerInvalidate(frame); + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); + attrs = PyStackRef_FromPyObjectSteal(attrs_o); + } + else { + if (_PyErr_Occurred(tstate)) { + JUMP_TO_LABEL(error); + } + attrs = PyStackRef_None; + } + s = subject; + tp = type; + n = names; + } + // _POP_TOP + { + value = n; + stack_pointer[-3] = attrs; + stack_pointer[-2] = s; + stack_pointer[-1] = tp; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = tp; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = s; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(MATCH_KEYS) { + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_KEYS; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_KEYS); + _PyStackRef subject; + _PyStackRef keys; + _PyStackRef values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, + PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + _PyFrame_StackPointerInvalidate(frame); + if (values_or_none_o == NULL) { + JUMP_TO_LABEL(error); + } + values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); + stack_pointer[0] = values_or_none; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(MATCH_MAPPING) { + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_MAPPING; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_MAPPING); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(MATCH_SEQUENCE) { + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_SEQUENCE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_SEQUENCE); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(NOP) { + #if _Py_TAIL_CALL_INTERP + int opcode = NOP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOP); + DISPATCH(); + } + + TARGET(NOT_TAKEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = NOT_TAKEN; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOT_TAKEN); + DISPATCH(); + } + + TARGET(POP_EXCEPT) { + #if _Py_TAIL_CALL_INTERP + int opcode = POP_EXCEPT; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_EXCEPT); + _PyStackRef exc_value; + exc_value = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_XSETREF(exc_info->exc_value, + PyStackRef_IsNone(exc_value) + ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(POP_ITER) { + #if _Py_TAIL_CALL_INTERP + int opcode = POP_ITER; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_ITER); + _PyStackRef iter; + _PyStackRef index_or_null; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(iter); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH(); + } + + TARGET(POP_JUMP_IF_FALSE) { + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_FALSE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(POP_JUMP_IF_NONE) { + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NONE); + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + { + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; + } + else { + b = PyStackRef_False; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = value; + value = b; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + } + } + // _POP_JUMP_IF_TRUE + { + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(POP_JUMP_IF_NOT_NONE) { + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_NOT_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + { + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; + } + else { + b = PyStackRef_False; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = value; + value = b; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + } + } + // _POP_JUMP_IF_FALSE + { + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(POP_JUMP_IF_TRUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_TRUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_TRUE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(POP_TOP) { + #if _Py_TAIL_CALL_INTERP + int opcode = POP_TOP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_TOP); + _PyStackRef value; + value = stack_pointer[-1]; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH(); + } + + TARGET(PUSH_EXC_INFO) { + #if _Py_TAIL_CALL_INTERP + int opcode = PUSH_EXC_INFO; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_EXC_INFO); + _PyStackRef exc; + _PyStackRef prev_exc; + _PyStackRef new_exc; + exc = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); + } + else { + prev_exc = PyStackRef_None; + } + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(PUSH_NULL) { + #if _Py_TAIL_CALL_INTERP + int opcode = PUSH_NULL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_NULL); + _PyStackRef res; + res = PyStackRef_NULL; + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(RAISE_VARARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = RAISE_VARARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RAISE_VARARGS); + _PyStackRef *args; + args = &stack_pointer[-oparg]; + assert(oparg < 3); + PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; + PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = do_raise(tstate, exc, cause); + _PyFrame_StackPointerInvalidate(frame); + if (err) { + assert(oparg == 0); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + JUMP_TO_LABEL(exception_unwind); + } + JUMP_TO_LABEL(error); + } + + TARGET(RERAISE) { + #if _Py_TAIL_CALL_INTERP + int opcode = RERAISE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RERAISE); + _PyStackRef *values; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); + assert(oparg >= 0 && oparg <= 2); + if (oparg) { + frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyStackRef_UntagInt(values[0]); + } + assert(exc && PyExceptionInstance_Check(exc)); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyErr_SetRaisedException(tstate, exc); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + JUMP_TO_LABEL(exception_unwind); + } + + TARGET(RESERVED) { + #if _Py_TAIL_CALL_INTERP + int opcode = RESERVED; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESERVED); + assert(0 && "Executing RESERVED instruction."); + Py_FatalError("Executing RESERVED instruction."); + DISPATCH(); + } + + TARGET(RESUME) { + #if _Py_TAIL_CALL_INTERP + int opcode = RESUME; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(RESUME); + PREDICTED_RESUME:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + _PyFrame_StackPointerInvalidate(frame); + if (bytecode == NULL) { + JUMP_TO_LABEL(error); + } + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + next_instr = frame->instr_ptr; + DISPATCH(); + } + #endif + } + // _MAYBE_INSTRUMENT + { + #ifdef Py_GIL_DISABLED + + int check_instrumentation = 1; + #else + int check_instrumentation = (tstate->tracing == 0); + #endif + if (check_instrumentation) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + _PyFrame_StackPointerInvalidate(frame); + if (err) { + JUMP_TO_LABEL(error); + } + next_instr = this_instr; + DISPATCH(); + } + } + } + // _QUICKEN_RESUME + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_Resume(this_instr, tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics(tstate); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + } + DISPATCH(); + } + + TARGET(RESUME_CHECK) { + #if _Py_TAIL_CALL_INTERP + int opcode = RESUME_CHECK; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(RESUME_CHECK); + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + if (eval_breaker != version) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } + #endif + DISPATCH(); + } + + TARGET(RESUME_CHECK_JIT) { + #if _Py_TAIL_CALL_INTERP + int opcode = RESUME_CHECK_JIT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(RESUME_CHECK_JIT); + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ + // _RESUME_CHECK + { + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + if (eval_breaker != version) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } + #endif + } + // _JIT + { + #ifdef _Py_TIER2 + bool is_resume = this_instr->op.code == RESUME_CHECK_JIT; + _Py_BackoffCounter counter = this_instr[1].counter; + if ((backoff_counter_triggers(counter) && + !IS_JIT_TRACING() && + (this_instr->op.code == JUMP_BACKWARD_JIT || is_resume)) && + next_instr->op.code != ENTER_EXECUTOR) { + _Py_CODEUNIT *insert_exec_at = this_instr; + for (int tmp = oparg; tmp > 255; tmp >>= 8) { + insert_exec_at--; + } + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, + is_resume ? insert_exec_at : next_instr, stack_pointer, 0, NULL, oparg, NULL); + if (succ) { + ENTER_TRACING(); + } + else { + this_instr[1].counter = restart_backoff_counter(counter); + } + } + else { + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + #endif + } + DISPATCH(); + } + + TARGET(RETURN_GENERATOR) { + #if _Py_TAIL_CALL_INTERP + int opcode = RETURN_GENERATOR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_GENERATOR); + _PyStackRef res; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject* func = + (PyFunctionObject*)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject* gen = (PyGenObject*)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + assert(STACK_LEVEL() <= 2); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame* gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame* prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + CI_SET_ADAPTIVE_INTERPRETER_ENABLED_STATE + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject*)gen); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(RETURN_VALUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = RETURN_VALUE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_VALUE); + _PyStackRef value; + _PyStackRef retval; + _PyStackRef res; + // _MAKE_HEAP_SAFE + { + value = stack_pointer[-1]; + value = PyStackRef_MakeHeapSafe(value); + } + // _RETURN_VALUE + { + retval = value; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyStackRef temp = retval; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(STACK_LEVEL() == 0); + DTRACE_FUNCTION_RETURN(); + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(frame->return_offset); + res = temp; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(SEND) { + #if _Py_TAIL_CALL_INTERP + int opcode = SEND; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND); + PREDICTED_SEND:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef receiver; + _PyStackRef null_or_index; + _PyStackRef v; + _PyStackRef retval; + // _SPECIALIZE_SEND + { + receiver = stack_pointer[-3]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_Send(receiver, next_instr); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(SEND); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + // _SEND + { + v = stack_pointer[-1]; + null_or_index = stack_pointer[-2]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + if (!IS_PEP523_HOOKED(tstate) && + (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && + gen_try_set_executing((PyGenObject *)receiver_o)) + { + PyGenObject *gen = (PyGenObject *)receiver_o; + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v)); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2u + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2u + oparg); + assert(gen_frame->previous == NULL); + gen_frame->previous = frame; + DISPATCH_INLINED(gen_frame); + } + if (!PyStackRef_IsNull(null_or_index) && PyStackRef_IsNone(v)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, receiver, &null_or_index); + _PyFrame_StackPointerInvalidate(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + JUMP_TO_LABEL(error); + } + JUMPBY(oparg); + stack_pointer[-2] = null_or_index; + DISPATCH(); + } + retval = item; + } + else { + PyObject *v_o = PyStackRef_AsPyObjectBorrow(v); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PySendResultPair res = _PyIter_Send(receiver_o, v_o); + _PyFrame_StackPointerInvalidate(frame); + if (res.kind == PYGEN_ERROR) { + JUMP_TO_LABEL(error); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(v); + _PyFrame_StackPointerInvalidate(frame); + retval = PyStackRef_FromPyObjectSteal(res.object); + if (res.kind == PYGEN_RETURN) { + JUMPBY(oparg); + } + stack_pointer += 1; + } + } + stack_pointer[-2] = null_or_index; + stack_pointer[-1] = retval; + DISPATCH(); + } + + TARGET(SEND_ASYNC_GEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = SEND_ASYNC_GEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_ASYNC_GEN); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef null_in; + _PyStackRef v; + _PyStackRef asend; + _PyStackRef null_out; + _PyStackRef retval; + /* Skip 1 cache entry */ + // _GUARD_3OS_ASYNC_GEN_ASEND + { + iter = stack_pointer[-3]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (!PyAsyncGenASend_CheckExact(iter_o)) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + } + // _SEND_ASYNC_GEN + { + v = stack_pointer[-1]; + null_in = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyAsyncGenASend_CheckExact(iter_o)); + PyObject *val = PyStackRef_AsPyObjectBorrow(v); + PyObject *retval_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PySendResult what = _PyAsyncGenASend_Send(iter_o, val, &retval_o); + _PyFrame_StackPointerInvalidate(frame); + if (what == PYGEN_ERROR) { + JUMP_TO_LABEL(error); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(v); + _PyFrame_StackPointerInvalidate(frame); + asend = iter; + null_out = null_in; + retval = PyStackRef_FromPyObjectSteal(retval_o); + if (what == PYGEN_RETURN) { + JUMPBY(oparg); + } + } + stack_pointer[-2] = asend; + stack_pointer[-1] = null_out; + stack_pointer[0] = retval; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(SEND_GEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = SEND_GEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_GEN); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef receiver; + _PyStackRef v; + _PyStackRef gen_frame; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + } + // _SEND_GEN_FRAME + { + v = stack_pointer[-1]; + receiver = stack_pointer[-3]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + if (!gen_try_set_executing((PyGenObject *)gen)) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + STAT_INC(SEND, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2u + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2u + oparg); + pushed_frame->previous = frame; + gen_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + CI_UPDATE_CALL_COUNT + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(SEND_VIRTUAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = SEND_VIRTUAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_VIRTUAL); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef val; + _PyStackRef nos; + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef none; + _PyStackRef next; + /* Skip 1 cache entry */ + // _GUARD_TOS_IS_NONE + { + val = stack_pointer[-1]; + if (!PyStackRef_IsNone(val)) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + } + // _GUARD_NOS_NOT_NULL + { + nos = stack_pointer[-2]; + if (PyStackRef_IsNull(nos)) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + } + // _SEND_VIRTUAL + { + none = val; + null_or_index = nos; + iter = stack_pointer[-3]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + Py_ssize_t index = PyStackRef_UntagInt(null_or_index); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index); + _PyFrame_StackPointerInvalidate(frame); + PyObject *next_o = next_index.object; + index = next_index.index; + if (next_o == NULL) { + if (index < 0) { + JUMP_TO_LABEL(error); + } + next = none; + JUMPBY(oparg); + DISPATCH(); + } + next = PyStackRef_FromPyObjectSteal(next_o); + null_or_index = PyStackRef_TagInt(index); + } + stack_pointer[-2] = null_or_index; + stack_pointer[-1] = next; + DISPATCH(); + } + + TARGET(SETUP_ANNOTATIONS) { + #if _Py_TAIL_CALL_INTERP + int opcode = SETUP_ANNOTATIONS; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SETUP_ANNOTATIONS); + if (LOCALS() == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when setting up annotations"); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + int err; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject* ann_dict = _PyMapping_GetOptionalItem2(LOCALS(), &_Py_ID(__annotations__), &err); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (ann_dict == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + ann_dict = PyDict_New(); + _PyFrame_StackPointerInvalidate(frame); + if (ann_dict == NULL) { + JUMP_TO_LABEL(error); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(ann_dict); + _PyFrame_StackPointerInvalidate(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + else { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(ann_dict); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(SET_ADD) { + #if _Py_TAIL_CALL_INTERP + int opcode = SET_ADD; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_ADD); + _PyStackRef set; + _PyStackRef v; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectSteal(v)); + _PyFrame_StackPointerInvalidate(frame); + if (err) { + JUMP_TO_LABEL(pop_1_error); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(SET_FUNCTION_ATTRIBUTE) { + #if _Py_TAIL_CALL_INTERP + int opcode = SET_FUNCTION_ATTRIBUTE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); + _PyStackRef attr_st; + _PyStackRef func_in; + _PyStackRef func_out; + func_in = stack_pointer[-1]; + attr_st = stack_pointer[-2]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); + PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); + func_out = func_in; + assert(PyFunction_Check(func)); + size_t offset = _Py_FunctionAttributeOffsets[oparg]; + assert(offset != 0); + PyObject **ptr = (PyObject **)(((char *)func) + offset); + assert(*ptr == NULL); + *ptr = attr; + stack_pointer[-2] = func_out; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(SET_UPDATE) { + #if _Py_TAIL_CALL_INTERP + int opcode = SET_UPDATE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_UPDATE); + _PyStackRef set; + _PyStackRef iterable; + _PyStackRef i; + _PyStackRef value; + // _SET_UPDATE + { + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(iterable)); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + i = iterable; + } + // _POP_TOP + { + value = i; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(STORE_ATTR) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR); + PREDICTED_STORE_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + _PyStackRef v; + _PyStackRef owner; + // _SPECIALIZE_STORE_ATTR + { + owner = stack_pointer[-1]; + v = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + if (!PyStackRef_IsNull(v)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_StoreAttr(owner, next_instr, name); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH_SAME_OPARG(); + } + } + OPCODE_DEFERRED_INC(STORE_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + /* Skip 3 cache entries */ + // _STORE_ATTR + { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(owner); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(v); + _PyFrame_StackPointerInvalidate(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(STORE_ATTR_INSTANCE_VALUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR_INSTANCE_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef value; + _PyStackRef owner; + _PyStackRef o; + /* Skip 1 cache entry */ + // _LOCK_OBJECT + { + value = stack_pointer[-1]; + if (!LOCK_OBJECT(PyStackRef_AsPyObjectBorrow(value))) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + // _GUARD_TYPE_VERSION_LOCKED + { + owner = value; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + } + // _GUARD_DORV_NO_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + } + // _STORE_ATTR_INSTANCE_VALUE + { + value = stack_pointer[-2]; + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + STAT_INC(STORE_ATTR, hit); + assert(_PyObject_GetManagedDict(owner_o) == NULL); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *old_value = *value_ptr; + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); + if (old_value == NULL) { + PyDictValues *values = _PyObject_InlineValues(owner_o); + Py_ssize_t index = value_ptr - values->values; + _PyDictValues_AddToInsertionOrder(values, index); + } + UNLOCK_OBJECT(owner_o); + o = owner; + stack_pointer[-2] = o; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_XDECREF(old_value); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = o; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(STORE_ATTR_SLOT) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR_SLOT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + _PyStackRef o; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + // _STORE_ATTR_SLOT + { + value = stack_pointer[-2]; + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!LOCK_OBJECT(owner_o)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + char *addr = (char *)owner_o + index; + STAT_INC(STORE_ATTR, hit); + PyObject *old_value = *(PyObject **)addr; + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); + o = owner; + stack_pointer[-2] = o; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_XDECREF(old_value); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = o; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(STORE_ATTR_WITH_HINT) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR_WITH_HINT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + _PyStackRef o; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + // _STORE_ATTR_WITH_HINT + { + value = stack_pointer[-2]; + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (dict == NULL) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + if (!LOCK_OBJECT(dict)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (hint >= (size_t)dict->ma_keys->dk_nentries || + dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + PyObject *old_value = ep->me_value; + if (old_value == NULL) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + _PyFrame_StackPointerInvalidate(frame); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); + STAT_INC(STORE_ATTR, hit); + o = owner; + stack_pointer[-2] = o; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_XDECREF(old_value); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP + { + value = o; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(STORE_DEREF) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_DEREF; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_DEREF); + _PyStackRef v; + v = stack_pointer[-1]; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(STORE_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_FAST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST); + _PyStackRef value; + _PyStackRef trash; + // _SWAP_FAST + { + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + } + // _POP_TOP + { + value = trash; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(STORE_FAST_LOAD_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_FAST_LOAD_FAST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); + _PyStackRef value1; + _PyStackRef value2; + value1 = stack_pointer[-1]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + _PyStackRef tmp = GETLOCAL(oparg1); + GETLOCAL(oparg1) = value1; + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[-1] = value2; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH(); + } + + TARGET(STORE_FAST_STORE_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_FAST_STORE_FAST; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_STORE_FAST); + _PyStackRef value2; + _PyStackRef value1; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + _PyStackRef tmp = GETLOCAL(oparg1); + GETLOCAL(oparg1) = value1; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + tmp = GETLOCAL(oparg2); + GETLOCAL(oparg2) = value2; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH(); + } + + TARGET(STORE_GLOBAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_GLOBAL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_GLOBAL); + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + int err; + if (PyStackRef_IsNull(v)) { + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + err = PyDict_Pop(GLOBALS(), name, NULL); + _PyFrame_StackPointerInvalidate(frame); + if (err == 0) { + err = -1; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + _PyFrame_StackPointerInvalidate(frame); + } + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(v); + _PyFrame_StackPointerInvalidate(frame); + } + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH(); + } + + TARGET(STORE_NAME) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_NAME; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_NAME); + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int error = _PyEval_StoreName(tstate, v, name, ns); + _PyFrame_StackPointerInvalidate(frame); + if (PyStackRef_IsNull(v)) { + } + else { + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(v); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += 1; + } + if (error) { + JUMP_TO_LABEL(pop_1_error); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(STORE_SLICE) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SLICE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_SLICE); + _PyStackRef v; + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + // _SPECIALIZE_STORE_SLICE + { + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(STORE_SLICE); + #endif /* ENABLE_SPECIALIZATION */ + } + // _STORE_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop), + Py_None); + _PyFrame_StackPointerInvalidate(frame); + int err; + if (slice == NULL) { + err = 1; + } + else { + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(slice); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += 2; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = container; + container = PyStackRef_NULL; + stack_pointer[-3] = container; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-4] = v; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -4; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (err) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(STORE_SUBSCR) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SUBSCR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR); + PREDICTED_STORE_SUBSCR:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef container; + _PyStackRef sub; + _PyStackRef v; + // _SPECIALIZE_STORE_SUBSCR + { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_StoreSubscr(container, sub, next_instr); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(STORE_SUBSCR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + // _STORE_SUBSCR + { + v = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-3] = v; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (err) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(STORE_SUBSCR_DICT) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SUBSCR_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef nos; + _PyStackRef value; + _PyStackRef dict_st; + _PyStackRef sub; + _PyStackRef st; + // _GUARD_NOS_DICT_STORE_SUBSCRIPT + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!Py_TYPE(o)->tp_as_mapping) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + if (Py_TYPE(o)->tp_as_mapping->mp_ass_subscript != _PyDict_StoreSubscript) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + } + /* Skip 1 cache entry */ + // _STORE_SUBSCR_DICT + { + sub = stack_pointer[-1]; + dict_st = nos; + value = stack_pointer[-3]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(Py_TYPE(dict)->tp_as_mapping->mp_ass_subscript == _PyDict_StoreSubscript); + STAT_INC(STORE_SUBSCR, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(sub), + PyStackRef_AsPyObjectSteal(value)); + _PyFrame_StackPointerInvalidate(frame); + if (err) { + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(dict_st); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + st = dict_st; + } + // _POP_TOP + { + value = st; + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(STORE_SUBSCR_LIST_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SUBSCR_LIST_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef list_st; + _PyStackRef sub_st; + _PyStackRef ls; + _PyStackRef ss; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + } + // _GUARD_NOS_LIST + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + } + /* Skip 1 cache entry */ + // _STORE_SUBSCR_LIST_INT + { + sub_st = value; + list_st = nos; + value = stack_pointer[-3]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + if (!LOCK_OBJECT(list)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + Py_ssize_t len = PyList_GET_SIZE(list); + if (index < 0) { + index += len; + } + if (index < 0 || index >= len) { + UNLOCK_OBJECT(list); + if (true) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + } + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], + PyStackRef_AsPyObjectSteal(value)); + assert(old_value != NULL); + UNLOCK_OBJECT(list); + ls = list_st; + ss = sub_st; + stack_pointer[-3] = ls; + stack_pointer[-2] = ss; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(old_value); + _PyFrame_StackPointerInvalidate(frame); + } + // _POP_TOP_INT + { + value = ss; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + // _POP_TOP + { + value = ls; + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(SWAP) { + #if _Py_TAIL_CALL_INTERP + int opcode = SWAP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SWAP); + _PyStackRef bottom; + _PyStackRef top; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + stack_pointer[-2 - (oparg-2)] = bottom; + stack_pointer[-1] = top; + DISPATCH(); + } + + TARGET(TO_BOOL) { + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL); + PREDICTED_TO_BOOL:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef value; + _PyStackRef res; + // _SPECIALIZE_TO_BOOL + { + value = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_ToBool(value, next_instr); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(TO_BOOL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + /* Skip 2 cache entries */ + // _TO_BOOL + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + res = err ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(TO_BOOL_ALWAYS_TRUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_ALWAYS_TRUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + } + // _REPLACE_WITH_TRUE + { + value = owner; + res = PyStackRef_True; + v = value; + } + // _POP_TOP + { + value = v; + stack_pointer[-1] = res; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(TO_BOOL_BOOL) { + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_BOOL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_BOOL); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + if (!PyStackRef_BoolCheck(value)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + DISPATCH(); + } + + TARGET(TO_BOOL_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_INT); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + // _GUARD_TOS_EXACT_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + } + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _TO_BOOL_INT + { + STAT_INC(TO_BOOL, hit); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + res = (_PyLong_IsZero((PyLongObject *)value_o)) ? PyStackRef_False : PyStackRef_True; + v = value; + } + // _POP_TOP_INT + { + value = v; + assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); + } + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(TO_BOOL_LIST) { + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_LIST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_LIST); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + // _GUARD_TOS_LIST + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + } + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _TO_BOOL_LIST + { + value = tos; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyList_CheckExact(value_o)); + STAT_INC(TO_BOOL, hit); + res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; + v = value; + } + // _POP_TOP + { + value = v; + stack_pointer[-1] = res; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(TO_BOOL_NONE) { + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_NONE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + if (!PyStackRef_IsNone(value)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(TO_BOOL_STR) { + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_STR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_STR); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + } + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _TO_BOOL_STR + { + STAT_INC(TO_BOOL, hit); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + res = value_o == &_Py_STR(empty) ? PyStackRef_False : PyStackRef_True; + v = value; + } + // _POP_TOP_UNICODE + { + value = v; + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + } + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(TRACE_RECORD) { + #if _Py_TAIL_CALL_INTERP + int opcode = TRACE_RECORD; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TRACE_RECORD); + opcode = TRACE_RECORD; + #if _Py_TIER2 + assert(IS_JIT_TRACING()); + next_instr = this_instr; + frame->instr_ptr = prev_instr; + opcode = next_instr->op.code; + bool stop_tracing = ( + opcode == WITH_EXCEPT_START || + opcode == RERAISE || + opcode == CLEANUP_THROW || + opcode == PUSH_EXC_INFO || + opcode == INTERPRETER_EXIT || + (opcode >= MIN_INSTRUMENTED_OPCODE && opcode != ENTER_EXECUTOR) + ); + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + _PyJitTracerState *tracer = _tstate->jit_tracer_state; + assert(tracer != NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); + _PyFrame_StackPointerInvalidate(frame); + if (full) { + LEAVE_TRACING(); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = stop_tracing_and_jit(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH(); + } + for (int i = 0; i < tracer->prev_state.recorded_count; i++) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_CLEAR(tracer->prev_state.recorded_values[i]); + _PyFrame_StackPointerInvalidate(frame); + } + tracer->prev_state.recorded_count = 0; + tracer->prev_state.instr = next_instr; + PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + if (tracer->prev_state.instr_code != (PyCodeObject *)prev_code) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_SETREF(tracer->prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); + _PyFrame_StackPointerInvalidate(frame); + } + tracer->prev_state.instr_frame = frame; + tracer->prev_state.instr_oparg = oparg; + tracer->prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + // Branch opcodes use the cache for branch history, not + // specialization counters. Don't reset it. + && !IS_CONDITIONAL_JUMP_OPCODE(opcode)) { + (&next_instr[1])->counter = trigger_backoff_counter(); + } + const _PyOpcodeRecordEntry *record_entry = &_PyOpcode_RecordEntries[opcode]; + for (int i = 0; i < record_entry->count; i++) { + _Py_RecordFuncPtr doesnt_escape = _PyOpcode_RecordFunctions[record_entry->indices[i]]; + doesnt_escape(frame, stack_pointer, oparg, &tracer->prev_state.recorded_values[i]); + } + tracer->prev_state.recorded_count = record_entry->count; + DISPATCH_GOTO_NON_TRACING(); + #else + (void)prev_instr; + Py_FatalError("JIT instruction executed in non-jit build."); + #endif + } + + TARGET(UNARY_INVERT) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNARY_INVERT; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_INVERT); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + // _UNARY_INVERT + { + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + v = value; + } + // _POP_TOP + { + value = v; + stack_pointer[-1] = res; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(UNARY_NEGATIVE) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNARY_NEGATIVE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NEGATIVE); + _PyStackRef value; + _PyStackRef res; + _PyStackRef v; + // _UNARY_NEGATIVE + { + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + v = value; + } + // _POP_TOP + { + value = v; + stack_pointer[-1] = res; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyStackRef_XCLOSE(value); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(UNARY_NOT) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNARY_NOT; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NOT); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + stack_pointer[-1] = res; + DISPATCH(); + } + + TARGET(UNPACK_EX) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_EX; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNPACK_EX); + _PyStackRef seq; + _PyStackRef *top; + seq = stack_pointer[-1]; + top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(seq_o); + _PyFrame_StackPointerInvalidate(frame); + if (res == 0) { + JUMP_TO_LABEL(error); + } + stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE); + PREDICTED_UNPACK_SEQUENCE:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef seq; + _PyStackRef *top; + // _SPECIALIZE_UNPACK_SEQUENCE + { + seq = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + (void)seq; + (void)counter; + } + // _UNPACK_SEQUENCE + { + top = &stack_pointer[-1 + oparg]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_DECREF(seq_o); + _PyFrame_StackPointerInvalidate(frame); + if (res == 0) { + JUMP_TO_LABEL(error); + } + } + stack_pointer += oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE_LIST) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE_LIST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef seq; + _PyStackRef *values; + // _GUARD_TOS_LIST + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + } + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_LIST + { + seq = tos; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyList_CheckExact(seq_o)); + if (!LOCK_OBJECT(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + if (PyList_GET_SIZE(seq_o) != oparg) { + UNLOCK_OBJECT(seq_o); + if (true) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyList_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + UNLOCK_OBJECT(seq_o); + stack_pointer += -1 + oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(seq); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE_TUPLE) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef seq; + _PyStackRef *values; + // _GUARD_TOS_TUPLE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + } + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_TUPLE + { + seq = tos; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != oparg) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + stack_pointer += -1 + oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(seq); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE_TWO_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + // _GUARD_TOS_TUPLE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + } + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_TWO_TUPLE + { + seq = tos; + assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != 2) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(seq); + _PyFrame_StackPointerInvalidate(frame); + } + DISPATCH(); + } + + TARGET(WITH_EXCEPT_START) { + #if _Py_TAIL_CALL_INTERP + int opcode = WITH_EXCEPT_START; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(WITH_EXCEPT_START); + _PyStackRef exit_func; + _PyStackRef exit_self; + _PyStackRef lasti; + _PyStackRef val; + _PyStackRef res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_self = stack_pointer[-4]; + exit_func = stack_pointer[-5]; + PyObject *exc, *tb; + PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); + PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); + assert(val_o && PyExceptionInstance_Check(val_o)); + exc = PyExceptionInstance_Class(val_o); + PyObject *original_tb = tb = PyException_GetTraceback(val_o); + if (tb == NULL) { + tb = Py_None; + } + assert(PyStackRef_IsTaggedInt(lasti)); + (void)lasti; + PyObject* res_o; + { + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + int has_self = !PyStackRef_IsNull(exit_self); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + _PyFrame_StackPointerInvalidate(frame); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + Py_XDECREF(original_tb); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(YIELD_VALUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = YIELD_VALUE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(YIELD_VALUE); + opcode = YIELD_VALUE; + _PyStackRef value; + _PyStackRef retval; + // _MAKE_HEAP_SAFE + { + value = stack_pointer[-1]; + value = PyStackRef_MakeHeapSafe(value); + } + // _YIELD_VALUE + { + retval = value; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + _PyStackRef temp = retval; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + DTRACE_FUNCTION_RETURN(); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; + FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE && defined(Py_DEBUG) + if (!PyStackRef_IsNone(frame->f_executable)) { + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; + assert(opcode == SEND || opcode == FOR_ITER); + } + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + value = temp; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + /* END INSTRUCTIONS */ +#if !_Py_TAIL_CALL_INTERP +#if USE_COMPUTED_GOTOS + _unknown_opcode: +#else + EXTRA_CASES // From pycore_opcode_metadata.h, a 'case' for each unused opcode +#endif + /* Tell C compilers not to hold the opcode variable in the loop. + next_instr points the current instruction without TARGET(). */ + opcode = next_instr->op.code; + _PyErr_Format(tstate, PyExc_SystemError, + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); +JUMP_TO_LABEL(error); + + + } + + /* This should never be reached. Every opcode should end with DISPATCH() + or goto error. */ + Py_UNREACHABLE(); +#endif /* _Py_TAIL_CALL_INTERP */ + /* BEGIN LABELS */ + + LABEL(pop_2_error) + { + stack_pointer -= 2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + + LABEL(pop_1_error) + { + stack_pointer -= 1; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + + LABEL(error) + { + _PyFrame_StackAssertInvalid(frame); + #ifdef NDEBUG + if (!_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyErr_SetString(tstate, PyExc_SystemError, + "error return without exception set"); + _PyFrame_StackPointerInvalidate(frame); + } + #else + assert(_PyErr_Occurred(tstate)); + #endif + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + STOP_TRACING(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + if (!_PyFrame_IsIncomplete(frame)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyFrameObject *f = _PyFrame_GetFrameObject(frame); + _PyFrame_StackPointerInvalidate(frame); + if (f != NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyTraceBack_Here(f); + _PyFrame_StackPointerInvalidate(frame); + } + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_MonitorRaise(tstate, frame, next_instr-1); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + JUMP_TO_LABEL(exception_unwind); + } + + LABEL(exception_unwind) + { + STOP_TRACING(); + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + int handled = get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti); + if (handled == 0) { + assert(_PyErr_Occurred(tstate)); + _PyStackRef *stackbase = _PyFrame_Stackbase(frame); + while (frame->stackpointer > stackbase) { + _PyStackRef ref = _PyFrame_StackPop(frame); + PyStackRef_XCLOSE(ref); + } + monitor_unwind(tstate, frame, next_instr-1); + JUMP_TO_LABEL(exit_unwind); + } + assert(STACK_LEVEL() >= level); + _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; + assert(frame->stackpointer >= new_top); + while (frame->stackpointer > new_top) { + _PyStackRef ref = _PyFrame_StackPop(frame); + PyStackRef_XCLOSE(ref); + } + if (lasti) { + int frame_lasti = _PyInterpreterFrame_LASTI(frame); + _PyStackRef lasti = PyStackRef_TagInt(frame_lasti); + _PyFrame_StackPush(frame, lasti); + } + PyObject *exc = _PyErr_GetRaisedException(tstate); + _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(exc)); + next_instr = _PyFrame_GetBytecode(frame) + handler; + int err = monitor_handled(tstate, frame, next_instr, exc); + if (err < 0) { + JUMP_TO_LABEL(exception_unwind); + } + #ifdef Py_DEBUG + if (frame->lltrace >= 5) { + lltrace_resume_frame(frame); + } + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + #if _Py_TAIL_CALL_INTERP + int opcode; + #endif + DISPATCH(); + } + + LABEL(exit_unwind) + { + assert(_PyErr_Occurred(tstate)); + DTRACE_FUNCTION_RETURN(); + JUMP_TO_LABEL(exit_unwind_notrace); + } + + LABEL(exit_unwind_notrace) + { + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { + tstate->current_frame = frame->previous; + #if !_Py_TAIL_CALL_INTERP + assert(frame == &entry.frame); + #endif + #ifdef _Py_TIER2 + _PyStackRef executor = frame->localsplus[0]; + assert(tstate->current_executor == NULL); + if (!PyStackRef_IsNull(executor)) { + tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); + PyStackRef_CLOSE(executor); + } + #endif + return NULL; + } + next_instr = frame->instr_ptr; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + + LABEL(start_frame) + { + CI_UPDATE_CALL_COUNT + int too_deep = _Py_EnterRecursivePy(tstate); + if (too_deep) { + JUMP_TO_LABEL(exit_unwind); + } + next_instr = frame->instr_ptr; + #ifdef Py_DEBUG + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + if (lltrace < 0) { + JUMP_TO_LABEL(exit_unwind); + } + frame->lltrace = lltrace; + assert(!_PyErr_Occurred(tstate)); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + #if _Py_TAIL_CALL_INTERP + int opcode; + #endif + DISPATCH(); + } + + #if _Py_TAIL_CALL_INTERP && !defined(_Py_TIER2) + Py_GCC_ATTRIBUTE((unused)) + #endif + LABEL(stop_tracing) + { + #if _Py_TIER2 + assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, _EXIT_TRACE); + _PyFrame_StackPointerInvalidate(frame); + LEAVE_TRACING(); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + int err = stop_tracing_and_jit(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH_GOTO_NON_TRACING(); + #else + Py_FatalError("JIT label executed in non-jit build."); + #endif + } + +/* END LABELS */ +#undef TIER_ONE diff --git a/cinderx/Interpreter/3.16/borrowed-ceval.c.template b/cinderx/Interpreter/3.16/borrowed-ceval.c.template new file mode 100644 index 000000000..f2c34954b --- /dev/null +++ b/cinderx/Interpreter/3.16/borrowed-ceval.c.template @@ -0,0 +1,32 @@ + +#include "internal/pycore_ceval.h" +#include "internal/pycore_stackref.h" +#include "internal/pycore_unicodeobject.h" +#include "internal/pycore_list.h" +#include "internal/pycore_long.h" +#include "internal/pycore_sliceobject.h" +#include "internal/pycore_stackref.h" +#include "internal/pycore_interpolation.h" +#include "internal/pycore_emscripten_signal.h" +#include "internal/pycore_template.h" +#include "internal/pycore_setobject.h" +#include "internal/pycore_intrinsics.h" +#include "internal/pycore_cell.h" +#include "internal/pycore_import.h" +#include "internal/pycore_range.h" +#include "internal/pycore_genobject.h" +#include "internal/pycore_function.h" +#include "internal/pycore_call.h" +#include "internal/pycore_floatobject.h" +#include "internal/pycore_tuple.h" + +#define _PyCoro_GetAwaitableIter JitCoro_GetAwaitableIter +#define _PyEval_GetAwaitable Ci_PyEval_GetAwaitable +#define _PyEval_GetANext Ci_PyEval_GetANext + +// @Borrow function _PyEval_GetANext from Python/ceval.c +// @Borrow function _PyEval_GetAwaitable from Python/ceval.c + +#undef _PyCoro_GetAwaitableIter +#undef _PyEval_GetAwaitable +#undef _PyEval_GetANext diff --git a/cinderx/Interpreter/3.16/ceval.h b/cinderx/Interpreter/3.16/ceval.h new file mode 100644 index 000000000..02533d49b --- /dev/null +++ b/cinderx/Interpreter/3.16/ceval.h @@ -0,0 +1,94 @@ +// @generated by UpstreamBorrow.py. +// See the Buck target fbcode//cinderx/UpstreamBorrow:gen_borrowed.c. + + +#include "internal/pycore_ceval.h" +#include "internal/pycore_stackref.h" +#include "internal/pycore_unicodeobject.h" +#include "internal/pycore_list.h" +#include "internal/pycore_long.h" +#include "internal/pycore_sliceobject.h" +#include "internal/pycore_stackref.h" +#include "internal/pycore_interpolation.h" +#include "internal/pycore_emscripten_signal.h" +#include "internal/pycore_template.h" +#include "internal/pycore_setobject.h" +#include "internal/pycore_intrinsics.h" +#include "internal/pycore_cell.h" +#include "internal/pycore_import.h" +#include "internal/pycore_range.h" +#include "internal/pycore_genobject.h" +#include "internal/pycore_function.h" +#include "internal/pycore_call.h" +#include "internal/pycore_floatobject.h" +#include "internal/pycore_tuple.h" + +#define _PyCoro_GetAwaitableIter JitCoro_GetAwaitableIter +#define _PyEval_GetAwaitable Ci_PyEval_GetAwaitable +#define _PyEval_GetANext Ci_PyEval_GetANext + +PyObject * +_PyEval_GetANext(PyObject *aiter) +{ + unaryfunc getter = NULL; + PyObject *next_iter = NULL; + PyTypeObject *type = Py_TYPE(aiter); + if (PyAsyncGen_CheckExact(aiter)) { + return type->tp_as_async->am_anext(aiter); + } + if (type->tp_as_async != NULL){ + getter = type->tp_as_async->am_anext; + } + + if (getter != NULL) { + next_iter = (*getter)(aiter); + if (next_iter == NULL) { + return NULL; + } + } + else { + PyErr_Format(PyExc_TypeError, + "'async for' requires an iterator with " + "__anext__ method, got %.100s", + type->tp_name); + return NULL; + } + + PyObject *awaitable = _PyCoro_GetAwaitableIter(next_iter); + if (awaitable == NULL) { + _PyErr_FormatFromCause( + PyExc_TypeError, + "'async for' received an invalid object " + "from __anext__: %.100s", + Py_TYPE(next_iter)->tp_name); + } + Py_DECREF(next_iter); + return awaitable; +} +PyObject * +_PyEval_GetAwaitable(PyObject *iterable, int oparg) +{ + PyObject *iter = _PyCoro_GetAwaitableIter(iterable); + + if (iter == NULL) { + _PyEval_FormatAwaitableError(PyThreadState_GET(), + Py_TYPE(iterable), oparg); + } + else if (PyCoro_CheckExact(iter)) { + PyCoroObject *coro = (PyCoroObject *)iter; + int8_t frame_state = FT_ATOMIC_LOAD_INT8_RELAXED(coro->cr_frame_state); + if (frame_state == FRAME_SUSPENDED_YIELD_FROM || + frame_state == FRAME_SUSPENDED_YIELD_FROM_LOCKED) + { + /* `iter` is a coroutine object that is being awaited. */ + Py_CLEAR(iter); + _PyErr_SetString(PyThreadState_GET(), PyExc_RuntimeError, + "coroutine is being awaited already"); + } + } + return iter; +} + +#undef _PyCoro_GetAwaitableIter +#undef _PyEval_GetAwaitable +#undef _PyEval_GetANext diff --git a/cinderx/Interpreter/3.16/cinder-bytecodes.c b/cinderx/Interpreter/3.16/cinder-bytecodes.c new file mode 100644 index 000000000..bd6076d02 --- /dev/null +++ b/cinderx/Interpreter/3.16/cinder-bytecodes.c @@ -0,0 +1,1130 @@ +// This file contains instruction definitions. +// It is read by generators stored in Tools/cases_generator/ +// to generate Python/generated_cases.c.h and others. +// Note that there is some dummy C code at the top and bottom of the file +// to fool text editors like VS Code into believing this is valid C code. +// The actual instruction definitions start at // BEGIN BYTECODES //. +// See Tools/cases_generator/README.md for more information. + +#include "Python.h" +#include "dictobject.h" +#include "opcode.h" +#include "optimizer.h" +#include "pycore_abstract.h" // _PyIndex_Check() +#include "pycore_audit.h" // _PySys_Audit() +#include "pycore_backoff.h" +#include "pycore_cell.h" // PyCell_GetRef() +#include "pycore_code.h" +#include "pycore_dict.h" +#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS +#include "pycore_frame.h" +#include "pycore_function.h" +#include "pycore_instruments.h" +#include "pycore_interpolation.h" // _PyInterpolation_Build() +#include "pycore_intrinsics.h" +#include "pycore_long.h" // _PyLong_ExactDealloc(), _PyLong_GetZero() +#include "pycore_moduleobject.h" // PyModuleObject +#include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_opcode_metadata.h" // uop names +#include "pycore_opcode_utils.h" // MAKE_FUNCTION_* +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() +#include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_range.h" // _PyRangeIterObject +#include "pycore_setobject.h" // _PySet_NextEntry() +#include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs +#include "pycore_stackref.h" +#include "pycore_template.h" // _PyTemplate_Build() +#include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_typeobject.h" // _PySuper_Lookup() +#include "pydtrace.h" +#include "setobject.h" + + +#include "cinderx/module_c_state.h" + +#define USE_COMPUTED_GOTOS 0 +#include "Python/ceval_macros.h" + +/* Flow control macros */ + +#define inst(name, ...) case name: +#define op(name, ...) /* NAME is ignored */ +#define macro(name) static int MACRO_##name +#define super(name) static int SUPER_##name +#define family(name, ...) static int family_##name +#define pseudo(name) static int pseudo_##name +#define label(name) \ + name: + +/* Annotations */ +#define guard +#define override +#define specializing +#define replicate(TIMES) +#define tier1 +#define no_save_ip + +// Dummy variables for stack effects. +static PyObject *value, *value1, *value2, *left, *right, *res, *sum, *prod, + *sub; +static PyObject *container, *start, *stop, *v, *lhs, *rhs, *res2; +static PyObject *list, *tuple, *dict, *owner, *set, *str, *tup, *map, *keys; +static PyObject *exit_func, *lasti, *val, *retval, *obj, *iter, *exhausted; +static PyObject *aiter, *awaitable, *iterable, *w, *exc_value, *bc, *locals; +static PyObject *orig, *excs, *update, *b, *fromlist, *level, *from; +static PyObject **pieces, **values; +static size_t jump; +// Dummy variables for cache effects +static uint16_t invert, counter, index, hint; +#define unused 0 // Used in a macro def, can't be static +static uint32_t type_version; +static _PyExecutorObject* current_executor; + +static PyObject* dummy_func( + PyThreadState* tstate, + _PyInterpreterFrame* frame, + unsigned char opcode, + unsigned int oparg, + _Py_CODEUNIT* next_instr, + PyObject** stack_pointer, + int throwflag, + PyObject* args[]) { +// Dummy labels. +pop_1_error: + // Dummy locals. + PyObject* dummy; + _Py_CODEUNIT* this_instr; + PyObject* attr; + PyObject* attrs; + PyObject* bottom; + PyObject* callable; + PyObject* callargs; + PyObject* codeobj; + PyObject* cond; + PyObject* descr; + PyObject* exc; + PyObject* exit; + PyObject* fget; + PyObject* fmt_spec; + PyObject* func; + uint32_t func_version; + PyObject* getattribute; + PyObject* kwargs; + PyObject* kwdefaults; + PyObject* len_o; + PyObject* match; + PyObject* match_type; + PyObject* method; + PyObject* mgr; + Py_ssize_t min_args; + PyObject* names; + PyObject* new_exc; + PyObject* next; + PyObject* none; + PyObject* null; + PyObject* prev_exc; + PyObject* receiver; + PyObject* rest; + int result; + PyObject* self; + PyObject* seq; + PyObject* slice; + PyObject* step; + PyObject* subject; + PyObject* top; + PyObject* type; + PyObject* typevars; + PyObject* val0; + PyObject* val1; + int values_or_none; + + switch (opcode) { + // BEGIN BYTECODES // + override inst(LOAD_COMMON_CONSTANT, ( -- value)) { + // Use our own copy of common constants to avoid depending on the + // offset of interp->common_consts within PyInterpreterState. + assert(oparg < NUM_COMMON_CONSTANTS); + value = PyStackRef_FromPyObjectNew(Ci_common_consts[oparg]); + } + + override op(_PUSH_FRAME, (new_frame--)) { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + DEAD(new_frame); + SAVE_STACK(); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + RELOAD_STACK(); + LOAD_IP(0); + + CI_UPDATE_CALL_COUNT + + LLTRACE_RESUME_FRAME(); + } + + override inst( + MAP_ADD, + (dict_st, unused[oparg - 1], key, value-- dict_st, unused[oparg - 1])) { + PyObject* dict = PyStackRef_AsPyObjectBorrow(dict_st); + /* dict[key] = value */ + int err = Ci_DictOrChecked_SetItem( + dict, + PyStackRef_AsPyObjectBorrow(key), + PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + PyStackRef_CLOSE(key); + ERROR_IF(err != 0); + } + + override inst( + LIST_APPEND, (list, unused[oparg - 1], v-- list, unused[oparg - 1])) { + int err = Ci_ListOrCheckedList_Append( + (PyListObject*)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectBorrow(v)); + PyStackRef_CLOSE(v); + ERROR_IF(err < 0); + } + + override inst(EXTENDED_OPCODE, (args[oparg >> 2]-- top[oparg & 0x03])) { + // Decode any extended oparg + int extop = (int)next_instr->op.code; + int extoparg = (int)next_instr->op.arg; + while (extop == EXTENDED_ARG) { + SKIP_OVER(1); + extoparg = extoparg << 8 | next_instr->op.arg; + extop = next_instr->op.code; + } + extop |= EXTENDED_OPCODE_FLAG; + + // Switch isn't supported in opcodes + if (extop == PRIMITIVE_LOAD_CONST) { + top[0] = PyStackRef_FromPyObjectNew( + PyTuple_GET_ITEM(GETITEM(FRAME_CO_CONSTS, extoparg), 0)); + DECREF_INPUTS(); + } else if (extop == STORE_LOCAL) { + _PyStackRef val = args[0]; + PyObject* local = GETITEM(FRAME_CO_CONSTS, extoparg); + int index = PyLong_AsInt(PyTuple_GET_ITEM(local, 0)); + int type = + _PyClassLoader_ResolvePrimitiveType(PyTuple_GET_ITEM(local, 1)); + + if (type < 0) { + DECREF_INPUTS(); + ERROR_IF(true); + } + + _PyStackRef tmp = GETLOCAL(index); + if (type == TYPED_DOUBLE) { + GETLOCAL(index) = PyStackRef_DUP(val); + } else { + Py_ssize_t ival = + unbox_primitive_int(PyStackRef_AsPyObjectBorrow(val)); + GETLOCAL(index) = + PyStackRef_FromPyObjectSteal(box_primitive(type, ival)); + } + + PyStackRef_XCLOSE(tmp); + +#if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) + if (adaptive_enabled) { + if (index < INT8_MAX && type < INT8_MAX) { + int16_t* cache = (int16_t*)next_instr; + *cache = (index << 8) | type; + _Ci_specialize(next_instr, STORE_LOCAL_CACHED); + } + } +#endif + DECREF_INPUTS(); + } else if (extop == LOAD_LOCAL) { + int index = PyLong_AsInt( + PyTuple_GET_ITEM(GETITEM(FRAME_CO_CONSTS, extoparg), 0)); + + _PyStackRef value = GETLOCAL(index); + if (PyStackRef_IsNull(value)) { + // Primitive values are default initialized to zero, so they don't + // need to be defined. We should consider stop doing that as it can + // cause compatibility issues when the same code runs statically and + // non statically. + GETLOCAL(index) = value = + PyStackRef_FromPyObjectSteal(PyLong_FromLong(0)); + } + value = PyStackRef_DUP(value); + DECREF_INPUTS(); + top[0] = value; + } else if (extop == PRIMITIVE_BOX) { + top[0] = sign_extend_primitive(args[0], extoparg); + DEAD(args); + } else if (extop == PRIMITIVE_UNBOX) { + PyObject* val = PyStackRef_AsPyObjectBorrow(args[0]); + if (PyLong_CheckExact(val)) { + size_t value; + int overflow = _PyClassLoader_CheckOverflow(val, extoparg, &value); + if (!overflow) { + PyErr_SetString(PyExc_OverflowError, "int overflow"); + DECREF_INPUTS(); + ERROR_IF(true); + } + } + DEAD(args); + } else if (extop == SEQUENCE_GET) { + PyObject* sequence = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject* idx = PyStackRef_AsPyObjectBorrow(args[1]); + PyObject* item; + Py_ssize_t val = (Py_ssize_t)PyLong_AsVoidPtr(idx); + + if (val == -1 && _PyErr_Occurred(tstate)) { + DECREF_INPUTS(); + ERROR_IF(true); + } + + // Adjust index + if (val < 0) { + val += Py_SIZE(sequence); + } + + extoparg &= ~SEQ_SUBSCR_UNCHECKED; + + if (extoparg == SEQ_LIST) { + item = PyList_GetItem(sequence, val); + if (item == NULL) { + DECREF_INPUTS(); + ERROR_IF(true); + } + Py_INCREF(item); + } else if (extoparg == SEQ_LIST_INEXACT) { + if (PyList_CheckExact(sequence) || + Py_TYPE(sequence)->tp_as_sequence->sq_item == + PyList_Type.tp_as_sequence->sq_item) { + item = PyList_GetItem(sequence, val); + if (item == NULL) { + DECREF_INPUTS(); + ERROR_IF(true); + } + Py_INCREF(item); + } else { + item = PyObject_GetItem(sequence, idx); + if (item == NULL) { + DECREF_INPUTS(); + ERROR_IF(true); + } + } + } else if (extoparg == SEQ_CHECKED_LIST) { + item = Ci_CheckedList_GetItem(sequence, val); + if (item == NULL) { + DECREF_INPUTS(); + ERROR_IF(true); + } + } else if (extoparg == SEQ_ARRAY_INT64) { + item = _Ci_StaticArray_Get(sequence, val); + if (item == NULL) { + DECREF_INPUTS(); + ERROR_IF(true); + } + } else { + PyErr_Format( + PyExc_SystemError, "bad oparg for SEQUENCE_GET: %d", extoparg); + DECREF_INPUTS(); + ERROR_IF(true); + } + + DECREF_INPUTS(); + top[0] = PyStackRef_FromPyObjectSteal(item); + } else if (extop == SEQUENCE_SET) { + PyObject* v = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject* sequence = PyStackRef_AsPyObjectBorrow(args[1]); + PyObject* subscr = PyStackRef_AsPyObjectBorrow(args[2]); + int err; + + Py_ssize_t idx = (Py_ssize_t)PyLong_AsVoidPtr(subscr); + + if (idx == -1 && _PyErr_Occurred(tstate)) { + DECREF_INPUTS(); + ERROR_IF(true); + } + + // Adjust index + if (idx < 0) { + idx += Py_SIZE(sequence); + } + + if (extoparg == SEQ_LIST) { + Py_INCREF(v); // PyList_SetItem steals the reference + err = PyList_SetItem(sequence, idx, v); + + if (err != 0) { + Py_DECREF(v); + DECREF_INPUTS(); + ERROR_IF(true); + } + } else if (extoparg == SEQ_LIST_INEXACT) { + if (PyList_CheckExact(sequence) || + Py_TYPE(sequence)->tp_as_sequence->sq_ass_item == + PyList_Type.tp_as_sequence->sq_ass_item) { + Py_INCREF(v); // PyList_SetItem steals the reference + err = PyList_SetItem(sequence, idx, v); + + if (err != 0) { + Py_DECREF(v); + DECREF_INPUTS(); + ERROR_IF(true); + } + } else { + err = PyObject_SetItem(sequence, subscr, v); + if (err != 0) { + DECREF_INPUTS(); + ERROR_IF(true); + } + } + } else if (extoparg == SEQ_ARRAY_INT64) { + err = _Ci_StaticArray_Set(sequence, idx, v); + + if (err != 0) { + DECREF_INPUTS(); + ERROR_IF(true); + } + } else { + PyErr_Format( + PyExc_SystemError, "bad oparg for SEQUENCE_SET: %d", oparg); + DECREF_INPUTS(); + ERROR_IF(true); + } + + DECREF_INPUTS(); + } else if (extop == FAST_LEN) { + PyObject* collection = PyStackRef_AsPyObjectBorrow(args[0]); + int inexact = extoparg & FAST_LEN_INEXACT; + extoparg &= ~FAST_LEN_INEXACT; + assert(FAST_LEN_LIST <= extoparg && extoparg <= FAST_LEN_STR); + PyObject* length; + if (inexact) { + // see if we have an exact type match, and if so, use the fastpath. + if ((extoparg == FAST_LEN_LIST && PyList_CheckExact(collection)) || + (extoparg == FAST_LEN_DICT && PyDict_CheckExact(collection)) || + (extoparg == FAST_LEN_SET && PyAnySet_CheckExact(collection)) || + (extoparg == FAST_LEN_TUPLE && PyTuple_CheckExact(collection)) || + (extoparg == FAST_LEN_ARRAY && + PyStaticArray_CheckExact(collection)) || + (extoparg == FAST_LEN_STR && PyUnicode_CheckExact(collection))) { + inexact = 0; + } + } + if (inexact) { + Py_ssize_t res = PyObject_Size(collection); + length = res >= 0 ? PyLong_FromSsize_t(res) : NULL; + } else if (extoparg == FAST_LEN_DICT) { + if (Ci_CheckedDict_Check(collection)) { + length = PyLong_FromLong(PyObject_Size(collection)); + } else { + assert(PyDict_Check(collection)); + length = PyLong_FromLong(((PyDictObject*)collection)->ma_used); + } + } else if (extoparg == FAST_LEN_SET) { + assert(PyAnySet_Check(collection)); + length = PyLong_FromLong(((PySetObject*)collection)->used); + } else { + // lists, tuples, arrays are all PyVarObject and use ob_size + assert( + PyTuple_Check(collection) || PyList_Check(collection) || + PyStaticArray_CheckExact(collection) || + PyUnicode_Check(collection) || Ci_CheckedList_Check(collection)); + length = PyLong_FromLong(Py_SIZE(collection)); + } + DECREF_INPUTS(); + ERROR_IF(length == NULL); + top[0] = PyStackRef_FromPyObjectSteal(length); + } else if (extop == LIST_DEL) { + PyObject* list = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject* subscr = PyStackRef_AsPyObjectBorrow(args[1]); + int err; + + Py_ssize_t idx = PyLong_AsLong(subscr); + + if (idx == -1 && _PyErr_Occurred(tstate)) { + DECREF_INPUTS(); + ERROR_IF(true); + } + + err = PyList_SetSlice(list, idx, idx + 1, NULL); + DECREF_INPUTS(); + ERROR_IF(err != 0); + } else if (extop == REFINE_TYPE) { + DEAD(args); + } else if (extop == LOAD_CLASS) { + PyObject* type_descr = GETITEM(FRAME_CO_CONSTS, extoparg); + int optional; + int exact; + PyObject* type = (PyObject*)_PyClassLoader_ResolveType( + type_descr, &optional, &exact); + DECREF_INPUTS(); + ERROR_IF(type == NULL); + top[0] = PyStackRef_FromPyObjectSteal(type); + } else if (extop == LOAD_TYPE) { + PyObject* instance = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject* type = (PyObject*)Py_TYPE(instance); + Py_INCREF(type); + DECREF_INPUTS(); + top[0] = PyStackRef_FromPyObjectSteal(type); + } else if (extop == BUILD_CHECKED_LIST) { + PyObject* list; + PyObject* list_info = GETITEM(FRAME_CO_CONSTS, extoparg); + PyObject* list_type = PyTuple_GET_ITEM(list_info, 0); + Py_ssize_t list_size = PyLong_AsLong(PyTuple_GET_ITEM(list_info, 1)); + + int optional; + int exact; + PyTypeObject* type = + _PyClassLoader_ResolveType(list_type, &optional, &exact); + assert(!optional); + +#if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) + if (adaptive_enabled) { + specialize_with_value( + next_instr, (PyObject*)type, BUILD_CHECKED_LIST_CACHED, 0, 0); + } +#endif + + list = Ci_CheckedList_New(type, list_size); + Py_DECREF(type); + + if (list == NULL) { + DECREF_INPUTS(); + ERROR_IF(true); + } + + for (Py_ssize_t i = 0; i < list_size; i++) { + Ci_ListOrCheckedList_SET_ITEM( + list, i, PyStackRef_AsPyObjectBorrow(args[i])); + } + DECREF_INPUTS(); + top[0] = PyStackRef_FromPyObjectSteal(list); + } else if (extop == BUILD_CHECKED_MAP) { + PyObject* map_info = GETITEM(FRAME_CO_CONSTS, extoparg); + PyObject* map_type = PyTuple_GET_ITEM(map_info, 0); + Py_ssize_t map_size = PyLong_AsLong(PyTuple_GET_ITEM(map_info, 1)); + + int optional; + int exact; + PyTypeObject* type = + _PyClassLoader_ResolveType(map_type, &optional, &exact); + if (type == NULL) { + DECREF_INPUTS(); + ERROR_IF(true); + } + assert(!optional); + +#if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) + if (adaptive_enabled) { + specialize_with_value( + next_instr, (PyObject*)type, BUILD_CHECKED_MAP_CACHED, 0, 0); + } +#endif + + PyObject* map = Ci_CheckedDict_NewPresized(type, map_size); + Py_DECREF(type); + if (map == NULL) { + DECREF_INPUTS(); + ERROR_IF(true); + } + + if (ci_build_dict(args, map_size, map) < 0) { + Py_DECREF(map); + DECREF_INPUTS(); + ERROR_IF(true); + } + DECREF_INPUTS(); + top[0] = PyStackRef_FromPyObjectSteal(map); + } else if (extop == LOAD_METHOD_STATIC) { + PyObject* self = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject* value = GETITEM(FRAME_CO_CONSTS, extoparg); + PyObject* target = PyTuple_GET_ITEM(value, 0); + int is_classmethod = _PyClassLoader_IsClassMethodDescr(value); + + Py_ssize_t slot = _PyClassLoader_ResolveMethod(target); + if (slot == -1) { + DECREF_INPUTS(); + ERROR_IF(true); + } + +#if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) + if (adaptive_enabled) { + // We encode class method as the low bit hence the >> 1. + if (slot < (INT32_MAX >> 1)) { + /* We smuggle in the information about whether the invocation was a + * classmethod in the low bit of the oparg. This is necessary, as + * without, the runtime won't be able to get the correct vtable from + * self when the type is passed in. + */ + int32_t* cache = (int32_t*)next_instr; + *cache = load_method_static_cached_oparg(slot, is_classmethod); + _Ci_specialize(next_instr, LOAD_METHOD_STATIC_CACHED); + } + } +#endif + + _PyType_VTable* vtable; + if (is_classmethod) { + vtable = (_PyType_VTable*)(((PyTypeObject*)self)->tp_cache); + } else { + vtable = (_PyType_VTable*)self->ob_type->tp_cache; + } + + assert(!PyErr_Occurred()); + StaticMethodInfo res = + _PyClassLoader_LoadStaticMethod(vtable, slot, self); + if (res.lmr_func == NULL) { + DECREF_INPUTS(); + ERROR_IF(true); + } + + _PyStackRef self_ref = PyStackRef_DUP(args[0]); + DECREF_INPUTS(); + top[0] = PyStackRef_FromPyObjectSteal(res.lmr_func); + top[1] = self_ref; + } else if (extop == INVOKE_METHOD) { + PyObject* target = PyStackRef_AsPyObjectBorrow(args[0]); + Py_ssize_t nargs = (oparg >> 2) - 1; + + assert(!PyErr_Occurred()); + + STACKREFS_TO_PYOBJECTS(&args[1], nargs, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true); + } + PyObject* res = PyObject_Vectorcall(target, args_o, nargs, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + DECREF_INPUTS(); + ERROR_IF(res == NULL); + top[0] = PyStackRef_FromPyObjectSteal(res); + } else if (extop == INVOKE_FUNCTION) { + // We should move to encoding the number of args directly in the + // opcode, right now pulling them out via invoke_function_args is a + // little ugly. + PyObject* value = GETITEM(FRAME_CO_CONSTS, extoparg); + int nargs = oparg >> 2; + PyObject* target = PyTuple_GET_ITEM(value, 0); + PyObject* container; + PyObject* func = _PyClassLoader_ResolveFunction(target, &container); + if (func == NULL) { + DECREF_INPUTS(); + ERROR_IF(true); + } + + STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true); + } + PyObject* res = _PyObject_Vectorcall(func, args_o, nargs, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); +#if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) + if (adaptive_enabled) { + if (_PyClassLoader_IsImmutable(container)) { + /* frozen type, we don't need to worry about indirecting */ + specialize_with_value( + next_instr, func, INVOKE_FUNCTION_CACHED, 0, 0); + } else { + PyObject** funcptr = _PyClassLoader_ResolveIndirectPtr(target); + PyObject*** cache = (PyObject***)next_instr; + *cache = funcptr; + _Ci_specialize(next_instr, INVOKE_INDIRECT_CACHED); + } + } +#endif + Py_DECREF(func); + Py_DECREF(container); + DECREF_INPUTS(); + ERROR_IF(res == NULL); + top[0] = PyStackRef_FromPyObjectSteal(res); + } else if (extop == INVOKE_NATIVE) { + PyObject* value = GETITEM(FRAME_CO_CONSTS, extoparg); + assert(PyTuple_CheckExact(value)); + Py_ssize_t nargs = oparg >> 2; + + PyObject* target = PyTuple_GET_ITEM(value, 0); + PyObject* name = PyTuple_GET_ITEM(target, 0); + PyObject* symbol = PyTuple_GET_ITEM(target, 1); + PyObject* signature = PyTuple_GET_ITEM(value, 1); + + STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + if (CONVERSION_FAILED(args_o)) { + DECREF_INPUTS(); + ERROR_IF(true); + } + + PyObject* res = _PyClassloader_InvokeNativeFunction( + name, symbol, signature, args_o, nargs); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + DECREF_INPUTS(); + ERROR_IF(res == NULL); + top[0] = PyStackRef_FromPyObjectSteal(res); + } else if (extop == TP_ALLOC) { + int optional; + int exact; + PyTypeObject* type = _PyClassLoader_ResolveType( + GETITEM(FRAME_CO_CONSTS, extoparg), &optional, &exact); + assert(!optional); + if (type == NULL) { + DECREF_INPUTS(); + ERROR_IF(true); + } + + PyObject* inst = type->tp_alloc(type, 0); + +#if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) + if (adaptive_enabled) { + specialize_with_value(next_instr, func, TP_ALLOC_CACHED, 0, 0); + } +#endif + Py_DECREF(type); + DECREF_INPUTS(); + ERROR_IF(inst == NULL); + top[0] = PyStackRef_FromPyObjectSteal(inst); + } else if (extop == CAST) { + PyObject* val = PyStackRef_AsPyObjectBorrow(args[0]); + int optional; + int exact; + PyTypeObject* type = _PyClassLoader_ResolveType( + GETITEM(FRAME_CO_CONSTS, extoparg), &optional, &exact); + if (type == NULL) { + DECREF_INPUTS(); + ERROR_IF(true); + } +#if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) + if (adaptive_enabled) { + specialize_with_value( + next_instr, + (PyObject*)type, + CAST_CACHED, + 2, + (exact << 1) | optional); + } +#endif + _PyStackRef res; + if (!_PyObject_TypeCheckOptional(val, type, optional, exact)) { + if (type == &PyFloat_Type && PyObject_TypeCheck(val, &PyLong_Type)) { + double dval = PyLong_AsDouble(val); + if (dval == -1.0 && PyErr_Occurred()) { + DECREF_INPUTS(); + ERROR_IF(true); + } + PyObject* fval = PyFloat_FromDouble(dval); + if (fval == NULL) { + DECREF_INPUTS(); + ERROR_IF(true); + } + res = PyStackRef_FromPyObjectSteal(fval); + } else { + PyErr_Format( + PyExc_TypeError, + exact ? "expected exactly '%s', got '%s'" + : "expected '%s', got '%s'", + type->tp_name, + Py_TYPE(val)->tp_name); + Py_DECREF(type); + DECREF_INPUTS(); + ERROR_IF(true); + } + } else { + res = PyStackRef_FromPyObjectNew(val); + } + + Py_DECREF(type); + DECREF_INPUTS(); + top[0] = res; + } else if (extop == PRIMITIVE_UNARY_OP) { + PyObject* res = + primitive_unary_op(PyStackRef_AsPyObjectBorrow(args[0]), extoparg); + DECREF_INPUTS(); + ERROR_IF(res == NULL); + top[0] = PyStackRef_FromPyObjectSteal(res); + } else if (extop == PRIMITIVE_BINARY_OP) { + PyObject* res = primitive_binary_op( + PyStackRef_AsPyObjectBorrow(args[0]), + PyStackRef_AsPyObjectBorrow(args[1]), + extoparg); + DECREF_INPUTS(); + ERROR_IF(res == NULL); + top[0] = PyStackRef_FromPyObjectSteal(res); + } else if (extop == PRIMITIVE_COMPARE_OP) { + PyObject* res = primitive_compare_op( + PyStackRef_AsPyObjectBorrow(args[0]), + PyStackRef_AsPyObjectBorrow(args[1]), + extoparg); + DECREF_INPUTS(); + ERROR_IF(res == NULL); + top[0] = PyStackRef_FromPyObjectSteal(res); + } else if (extop == LOAD_FIELD) { + PyObject* self = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject* field = GETITEM(FRAME_CO_CONSTS, extoparg); + PyObject* value; + int field_type; + Py_ssize_t offset = + _PyClassLoader_ResolveFieldOffset(field, &field_type); + if (offset == -1) { + DECREF_INPUTS(); + ERROR_IF(true); + } + + if (field_type == TYPED_OBJECT) { + value = *FIELD_OFFSET(self, offset); +#if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) + if (adaptive_enabled) { + if (offset < INT32_MAX) { + int32_t* cache = (int32_t*)next_instr; + *cache = offset; + _Ci_specialize(next_instr, LOAD_OBJ_FIELD); + } + } +#endif + + if (value == NULL) { + PyObject* name = + PyTuple_GET_ITEM(field, PyTuple_GET_SIZE(field) - 1); + PyErr_Format( + PyExc_AttributeError, + "'%.50s' object has no attribute '%U'", + Py_TYPE(self)->tp_name, + name); + DECREF_INPUTS(); + ERROR_IF(true); + } + Py_INCREF(value); + } else { +#if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) + if (adaptive_enabled) { + if (offset <= INT32_MAX >> 8) { + assert(field_type < 0xff); + int32_t* cache = (int32_t*)next_instr; + *cache = offset << 8 | field_type; + _Ci_specialize(next_instr, LOAD_PRIMITIVE_FIELD); + } + } +#endif + + value = load_field(field_type, (char*)FIELD_OFFSET(self, offset)); + if (value == NULL) { + DECREF_INPUTS(); + ERROR_IF(true); + } + } + DECREF_INPUTS(); + top[0] = PyStackRef_FromPyObjectSteal(value); + } else if (extop == STORE_FIELD) { + PyObject* value = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject* self = PyStackRef_AsPyObjectBorrow(args[1]); + PyObject* field = GETITEM(FRAME_CO_CONSTS, extoparg); + int field_type; + Py_ssize_t offset = + _PyClassLoader_ResolveFieldOffset(field, &field_type); + if (offset == -1) { + DECREF_INPUTS(); + ERROR_IF(true); + } + + PyObject** addr = FIELD_OFFSET(self, offset); + + if (field_type == TYPED_OBJECT) { + Py_INCREF(value); + Py_XDECREF(*addr); + *addr = value; +#if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) + if (adaptive_enabled) { + if (offset <= INT32_MAX) { + int32_t* cache = (int32_t*)next_instr; + *cache = offset; + _Ci_specialize(next_instr, STORE_OBJ_FIELD); + } + } +#endif + } else { +#if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) + if (adaptive_enabled) { + if (offset <= INT32_MAX >> 8) { + assert(field_type < 0xff); + int32_t* cache = (int32_t*)next_instr; + *cache = offset << 8 | field_type; + _Ci_specialize(next_instr, STORE_PRIMITIVE_FIELD); + } + } +#endif + store_field(field_type, (char*)addr, value); + } + DECREF_INPUTS(); + } else if (extop == RETURN_PRIMITIVE) { + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyStackRef temp = + sign_extend_primitive(PyStackRef_MakeHeapSafe(args[0]), extoparg); + DEAD(args); + SAVE_STACK(); + assert(STACK_LEVEL() == 0); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame* dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + RELOAD_STACK(); + LOAD_IP(frame->return_offset); + stack_pointer[0] = temp; + stack_pointer += 1; + LLTRACE_RESUME_FRAME(); + DISPATCH(); + } else if (extop == POP_JUMP_IF_ZERO) { + PyObject* cond = PyStackRef_AsPyObjectBorrow(args[0]); + int is_nonzero = PyObject_IsTrue(cond); + DECREF_INPUTS(); + SKIP_OVER(2); // skip cache + EXTENDED_OPCODE + if (!is_nonzero) { + JUMPBY(extoparg); + DISPATCH(); + } + DISPATCH(); + } else if (extop == POP_JUMP_IF_NONZERO) { + PyObject* cond = PyStackRef_AsPyObjectBorrow(args[0]); + int is_nonzero = PyObject_IsTrue(cond); + DECREF_INPUTS(); + SKIP_OVER(2); // skip cache and EXTENDED_OPCODE + if (is_nonzero) { + JUMPBY(extoparg); + DISPATCH(); + } + DISPATCH(); + } else if (extop == CONVERT_PRIMITIVE) { + PyObject* val = PyStackRef_AsPyObjectBorrow(args[0]); + Py_ssize_t from_type = extoparg & 0xFF; + Py_ssize_t to_type = extoparg >> 4; + Py_ssize_t extend_sign = + (from_type & TYPED_INT_SIGNED) && (to_type & TYPED_INT_SIGNED); + int size = to_type >> 1; + size_t ival = (size_t)PyLong_AsVoidPtr(val); + + ival &= trunc_masks[size]; + + // Extend the sign if needed + if (extend_sign != 0 && (ival & signed_bits[size])) { + ival |= (signex_masks[size]); + } + + PyObject* res = PyLong_FromSize_t(ival); + DECREF_INPUTS(); + ERROR_IF(res == NULL); + top[0] = PyStackRef_FromPyObjectSteal(res); + } else if (extop == LOAD_ITERABLE_ARG) { + PyObject* tup = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject* element; + int idx = extoparg; + _PyStackRef new_tup; + if (!PyTuple_CheckExact(tup)) { + if (tup->ob_type->tp_iter == NULL && !PySequence_Check(tup)) { + PyErr_Format( + PyExc_TypeError, + "argument after * " + "must be an iterable, not %.200s", + tup->ob_type->tp_name); + DECREF_INPUTS(); + ERROR_IF(true); + } + tup = PySequence_Tuple(tup); + if (tup == NULL) { + DECREF_INPUTS(); + ERROR_IF(true); + } + new_tup = PyStackRef_FromPyObjectSteal(tup); + } else { + new_tup = PyStackRef_FromPyObjectNew(tup); + } + + element = PyTuple_GetItem(tup, idx); + if (element == NULL) { + PyStackRef_CLOSE(new_tup); + DECREF_INPUTS(); + ERROR_IF(true); + } + Py_INCREF(element); + DECREF_INPUTS(); + top[0] = PyStackRef_FromPyObjectSteal(element); + top[1] = new_tup; + } else if (extop == LOAD_MAPPING_ARG) { + PyObject *defaultval, *mapping, *name; + if (extoparg == 3) { + defaultval = PyStackRef_AsPyObjectBorrow(args[0]); + mapping = PyStackRef_AsPyObjectBorrow(args[1]); + name = PyStackRef_AsPyObjectBorrow(args[2]); + } else { + defaultval = NULL; + mapping = PyStackRef_AsPyObjectBorrow(args[0]); + name = PyStackRef_AsPyObjectBorrow(args[1]); + } + PyObject* value; + if (!PyDict_Check(mapping) && !Ci_CheckedDict_Check(mapping)) { + PyErr_Format( + PyExc_TypeError, + "argument after ** " + "must be a dict, not %.200s", + mapping->ob_type->tp_name); + DECREF_INPUTS(); + ERROR_IF(true); + } + + value = PyDict_GetItemWithError(mapping, name); + if (value == NULL) { + if (_PyErr_Occurred(tstate)) { + DECREF_INPUTS(); + ERROR_IF(true); + } else if (oparg == 2) { + PyErr_Format(PyExc_TypeError, "missing argument %U", name); + assert(defaultval == NULL); + DECREF_INPUTS(); + ERROR_IF(true); + } else { + /* Default value is on the stack */ + value = defaultval; + } + } + + Py_INCREF(value); + DECREF_INPUTS(); + top[0] = PyStackRef_FromPyObjectSteal(value); + } else { + PyErr_Format( + PyExc_RuntimeError, "unsupported extended opcode: %d", extop); + DECREF_INPUTS(); + ERROR_IF(true); + } + SKIP_OVER(1); + } + + override inst(RETURN_VALUE, (retval-- res)) { + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); + DEAD(retval); + SAVE_STACK(); + assert(STACK_LEVEL() == 0); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame* dying = frame; + frame = tstate->current_frame = dying->previous; + + // CX: Maybe reactivate adaptive interpreter in caller + CI_SET_ADAPTIVE_INTERPRETER_ENABLED_STATE + + _PyEval_FrameClearAndPop(tstate, dying); + RELOAD_STACK(); + LOAD_IP(frame->return_offset); + res = temp; + LLTRACE_RESUME_FRAME(); + } + + override inst(RETURN_GENERATOR, (--res)) { + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject* func = + (PyFunctionObject*)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyGenObject* gen = (PyGenObject*)_Py_MakeCoro(func); + ERROR_IF(gen == NULL); + assert(STACK_LEVEL() <= 2); + SAVE_STACK(); + _PyInterpreterFrame* gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame* prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + + // CX: Maybe reactivate adaptive interpreter in caller + CI_SET_ADAPTIVE_INTERPRETER_ENABLED_STATE + + LOAD_IP(frame->return_offset); + RELOAD_STACK(); + res = PyStackRef_FromPyObjectStealMortal((PyObject*)gen); + LLTRACE_RESUME_FRAME(); + } + + override inst(YIELD_VALUE, (retval-- value)) { + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + frame->instr_ptr++; + PyGenObject* gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + _PyStackRef temp = retval; + DEAD(retval); + SAVE_STACK(); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame* gen_frame = frame; + frame = tstate->current_frame = frame->previous; + + // CX: Maybe reactivate adaptive interpreter in caller + CI_SET_ADAPTIVE_INTERPRETER_ENABLED_STATE + + gen_frame->previous = NULL; + ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; + FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); +#if TIER_ONE + assert( + frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); +#endif + RELOAD_STACK(); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + value = PyStackRef_MakeHeapSafe(temp); + LLTRACE_RESUME_FRAME(); + } + + spilled label(start_frame) { + CI_UPDATE_CALL_COUNT + + int too_deep = _Py_EnterRecursivePy(tstate); + if (too_deep) { + goto exit_unwind; + } + next_instr = frame->instr_ptr; +#ifdef Py_DEBUG + int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); + if (lltrace < 0) { + JUMP_TO_LABEL(exit_unwind); + } + frame->lltrace = lltrace; + /* _PyEval_EvalFrameDefault() must not be called with an exception set, + because it can clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!_PyErr_Occurred(tstate)); +#endif + RELOAD_STACK(); +#if _Py_TAIL_CALL_INTERP + int opcode; +#endif + DISPATCH(); + } + + // END BYTECODES // + } +dispatch_opcode: +error: +exception_unwind: +exit_unwind: +handle_eval_breaker: +resume_frame: +start_frame: +unbound_local_error:; +} + +// Future families go below this point // diff --git a/cinderx/Interpreter/3.16/cinder_opcode.h b/cinderx/Interpreter/3.16/cinder_opcode.h new file mode 100644 index 000000000..dfbe719a0 --- /dev/null +++ b/cinderx/Interpreter/3.16/cinder_opcode.h @@ -0,0 +1,29 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#include "cinderx/python.h" + +#define _PyOpcode_num_popped _CiOpcode_num_popped +#define _PyOpcode_num_pushed _CiOpcode_num_pushed +#define _PyOpcode_opcode_metadata _CiOpcode_opcode_metadata +#define _PyOpcode_macro_expansion _CiOpcode_macro_expansion +#define _PyOpcode_OpName _CiOpcode_OpName +#define _PyOpcode_Caches _CiOpcode_Caches +#define _PyOpcode_Deopt _CiOpcode_Deopt +#define _PyOpcode_PseudoTargets _CiOpcode_PseudoTargets + +#include "internal/pycore_opcode_metadata.h" + +#ifndef Ci_INTERNAL_OPCODE_H +#define Ci_INTERNAL_OPCODE_H + +#ifndef EXTENDED_OPCODE +#define EXTENDED_OPCODE 126 +#endif + +#ifndef EAGER_IMPORT_NAME +#define EAGER_IMPORT_NAME 121 +#endif + +#include "cinderx/Interpreter/cinder_opcode_ids.h" + +#endif diff --git a/cinderx/Interpreter/3.16/cinder_opcode_ids.h b/cinderx/Interpreter/3.16/cinder_opcode_ids.h new file mode 100644 index 000000000..ad12f6757 --- /dev/null +++ b/cinderx/Interpreter/3.16/cinder_opcode_ids.h @@ -0,0 +1,54 @@ + +// Copyright (c) Meta Platforms, Inc. and affiliates. + +// 3.16 has a simple file that just defines the relevant ids: + +#include "opcode.h" + +// 0x200 to make sure we don't collide with pseudo instructions +#define EXTENDED_OPCODE_FLAG 0x200 + + +#define INVOKE_METHOD (1 | EXTENDED_OPCODE_FLAG) +#define LOAD_FIELD (2 | EXTENDED_OPCODE_FLAG) +#define LOAD_OBJ_FIELD (5 | EXTENDED_OPCODE_FLAG) +#define LOAD_PRIMITIVE_FIELD (6 | EXTENDED_OPCODE_FLAG) +#define STORE_FIELD (7 | EXTENDED_OPCODE_FLAG) +#define STORE_OBJ_FIELD (8 | EXTENDED_OPCODE_FLAG) +#define STORE_PRIMITIVE_FIELD (9 | EXTENDED_OPCODE_FLAG) +#define BUILD_CHECKED_LIST (10 | EXTENDED_OPCODE_FLAG) +#define BUILD_CHECKED_LIST_CACHED (11 | EXTENDED_OPCODE_FLAG) +#define LOAD_TYPE (12 | EXTENDED_OPCODE_FLAG) +#define CAST (13 | EXTENDED_OPCODE_FLAG) +#define CAST_CACHED (14 | EXTENDED_OPCODE_FLAG) +#define LOAD_LOCAL (15 | EXTENDED_OPCODE_FLAG) +#define STORE_LOCAL (16 | EXTENDED_OPCODE_FLAG) +#define STORE_LOCAL_CACHED (17 | EXTENDED_OPCODE_FLAG) +#define PRIMITIVE_BOX (18 | EXTENDED_OPCODE_FLAG) +#define POP_JUMP_IF_ZERO (96 | EXTENDED_OPCODE_FLAG) +#define POP_JUMP_IF_NONZERO (99 | EXTENDED_OPCODE_FLAG) +#define PRIMITIVE_UNBOX (19 | EXTENDED_OPCODE_FLAG) +#define PRIMITIVE_BINARY_OP (20 | EXTENDED_OPCODE_FLAG) +#define PRIMITIVE_UNARY_OP (21 | EXTENDED_OPCODE_FLAG) +#define PRIMITIVE_COMPARE_OP (22 | EXTENDED_OPCODE_FLAG) +#define LOAD_ITERABLE_ARG (23 | EXTENDED_OPCODE_FLAG) +#define LOAD_MAPPING_ARG (24 | EXTENDED_OPCODE_FLAG) +#define INVOKE_FUNCTION (25 | EXTENDED_OPCODE_FLAG) +#define INVOKE_FUNCTION_CACHED (26 | EXTENDED_OPCODE_FLAG) +#define INVOKE_INDIRECT_CACHED (27 | EXTENDED_OPCODE_FLAG) +#define FAST_LEN (28 | EXTENDED_OPCODE_FLAG) +#define CONVERT_PRIMITIVE (29 | EXTENDED_OPCODE_FLAG) +#define INVOKE_NATIVE (30 | EXTENDED_OPCODE_FLAG) +#define LOAD_CLASS (31 | EXTENDED_OPCODE_FLAG) +#define BUILD_CHECKED_MAP (32 | EXTENDED_OPCODE_FLAG) +#define BUILD_CHECKED_MAP_CACHED (33 | EXTENDED_OPCODE_FLAG) +#define SEQUENCE_GET (34 | EXTENDED_OPCODE_FLAG) +#define SEQUENCE_SET (35 | EXTENDED_OPCODE_FLAG) +#define LIST_DEL (38 | EXTENDED_OPCODE_FLAG) +#define REFINE_TYPE (39 | EXTENDED_OPCODE_FLAG) +#define PRIMITIVE_LOAD_CONST (40 | EXTENDED_OPCODE_FLAG) +#define RETURN_PRIMITIVE (41 | EXTENDED_OPCODE_FLAG) +#define TP_ALLOC (43 | EXTENDED_OPCODE_FLAG) +#define TP_ALLOC_CACHED (44 | EXTENDED_OPCODE_FLAG) +#define LOAD_METHOD_STATIC (45 | EXTENDED_OPCODE_FLAG) +#define LOAD_METHOD_STATIC_CACHED (46 | EXTENDED_OPCODE_FLAG) diff --git a/build/fbcode_builder/getdeps/__init__.py b/cinderx/Interpreter/3.16/cinder_opcode_metadata.h similarity index 100% rename from build/fbcode_builder/getdeps/__init__.py rename to cinderx/Interpreter/3.16/cinder_opcode_metadata.h diff --git a/cinderx/Interpreter/3.16/cinderx_opcode_targets.h b/cinderx/Interpreter/3.16/cinderx_opcode_targets.h new file mode 100644 index 000000000..7e145aa83 --- /dev/null +++ b/cinderx/Interpreter/3.16/cinderx_opcode_targets.h @@ -0,0 +1,1293 @@ +#if !_Py_TAIL_CALL_INTERP +static void *opcode_targets_table[256] = { + &&TARGET_CACHE, + &&TARGET_BINARY_SLICE, + &&TARGET_BUILD_TEMPLATE, + &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, + &&TARGET_CALL_FUNCTION_EX, + &&TARGET_CHECK_EG_MATCH, + &&TARGET_CHECK_EXC_MATCH, + &&TARGET_CLEANUP_THROW, + &&TARGET_DELETE_SUBSCR, + &&TARGET_END_FOR, + &&TARGET_END_SEND, + &&TARGET_EXIT_INIT_CHECK, + &&TARGET_FORMAT_SIMPLE, + &&TARGET_FORMAT_WITH_SPEC, + &&TARGET_GET_AITER, + &&TARGET_GET_ANEXT, + &&TARGET_GET_LEN, + &&TARGET_RESERVED, + &&TARGET_INTERPRETER_EXIT, + &&TARGET_LOAD_BUILD_CLASS, + &&TARGET_LOAD_LOCALS, + &&TARGET_MAKE_FUNCTION, + &&TARGET_MATCH_KEYS, + &&TARGET_MATCH_MAPPING, + &&TARGET_MATCH_SEQUENCE, + &&TARGET_NOP, + &&TARGET_NOT_TAKEN, + &&TARGET_POP_EXCEPT, + &&TARGET_POP_ITER, + &&TARGET_POP_TOP, + &&TARGET_PUSH_EXC_INFO, + &&TARGET_PUSH_NULL, + &&TARGET_RETURN_GENERATOR, + &&TARGET_RETURN_VALUE, + &&TARGET_SETUP_ANNOTATIONS, + &&TARGET_STORE_SLICE, + &&TARGET_STORE_SUBSCR, + &&TARGET_TO_BOOL, + &&TARGET_UNARY_INVERT, + &&TARGET_UNARY_NEGATIVE, + &&TARGET_UNARY_NOT, + &&TARGET_WITH_EXCEPT_START, + &&TARGET_BINARY_OP, + &&TARGET_BUILD_INTERPOLATION, + &&TARGET_BUILD_LIST, + &&TARGET_BUILD_MAP, + &&TARGET_BUILD_SET, + &&TARGET_BUILD_SLICE, + &&TARGET_BUILD_STRING, + &&TARGET_BUILD_TUPLE, + &&TARGET_CALL, + &&TARGET_CALL_INTRINSIC_1, + &&TARGET_CALL_INTRINSIC_2, + &&TARGET_CALL_KW, + &&TARGET_COMPARE_OP, + &&TARGET_CONTAINS_OP, + &&TARGET_CONVERT_VALUE, + &&TARGET_COPY, + &&TARGET_COPY_FREE_VARS, + &&TARGET_DELETE_DEREF, + &&TARGET_DELETE_FAST, + &&TARGET_DICT_MERGE, + &&TARGET_DICT_UPDATE, + &&TARGET_END_ASYNC_FOR, + &&TARGET_EXTENDED_ARG, + &&TARGET_FOR_ITER, + &&TARGET_GET_AWAITABLE, + &&TARGET_GET_ITER, + &&TARGET_IMPORT_FROM, + &&TARGET_IMPORT_NAME, + &&TARGET_IS_OP, + &&TARGET_JUMP_BACKWARD, + &&TARGET_JUMP_BACKWARD_NO_INTERRUPT, + &&TARGET_JUMP_FORWARD, + &&TARGET_LIST_APPEND, + &&TARGET_LIST_EXTEND, + &&TARGET_LOAD_ATTR, + &&TARGET_LOAD_COMMON_CONSTANT, + &&TARGET_LOAD_CONST, + &&TARGET_LOAD_DEREF, + &&TARGET_LOAD_FAST, + &&TARGET_LOAD_FAST_AND_CLEAR, + &&TARGET_LOAD_FAST_BORROW, + &&TARGET_LOAD_FAST_BORROW_LOAD_FAST_BORROW, + &&TARGET_LOAD_FAST_CHECK, + &&TARGET_LOAD_FAST_LOAD_FAST, + &&TARGET_LOAD_FROM_DICT_OR_DEREF, + &&TARGET_LOAD_FROM_DICT_OR_GLOBALS, + &&TARGET_LOAD_GLOBAL, + &&TARGET_LOAD_NAME, + &&TARGET_LOAD_SMALL_INT, + &&TARGET_LOAD_SPECIAL, + &&TARGET_LOAD_SUPER_ATTR, + &&TARGET_MAKE_CELL, + &&TARGET_MAP_ADD, + &&TARGET_MATCH_CLASS, + &&TARGET_POP_JUMP_IF_FALSE, + &&TARGET_POP_JUMP_IF_NONE, + &&TARGET_POP_JUMP_IF_NOT_NONE, + &&TARGET_POP_JUMP_IF_TRUE, + &&TARGET_RAISE_VARARGS, + &&TARGET_RERAISE, + &&TARGET_SEND, + &&TARGET_SET_ADD, + &&TARGET_SET_FUNCTION_ATTRIBUTE, + &&TARGET_SET_UPDATE, + &&TARGET_STORE_ATTR, + &&TARGET_STORE_DEREF, + &&TARGET_STORE_FAST, + &&TARGET_STORE_FAST_LOAD_FAST, + &&TARGET_STORE_FAST_STORE_FAST, + &&TARGET_STORE_GLOBAL, + &&TARGET_STORE_NAME, + &&TARGET_SWAP, + &&TARGET_UNPACK_EX, + &&TARGET_UNPACK_SEQUENCE, + &&TARGET_YIELD_VALUE, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_EXTENDED_OPCODE, + &&_unknown_opcode, + &&TARGET_RESUME, + &&TARGET_BINARY_OP_ADD_FLOAT, + &&TARGET_BINARY_OP_ADD_INT, + &&TARGET_BINARY_OP_ADD_UNICODE, + &&TARGET_BINARY_OP_EXTEND, + &&TARGET_BINARY_OP_MULTIPLY_FLOAT, + &&TARGET_BINARY_OP_MULTIPLY_INT, + &&TARGET_BINARY_OP_SUBSCR_DICT, + &&TARGET_BINARY_OP_SUBSCR_GETITEM, + &&TARGET_BINARY_OP_SUBSCR_LIST_INT, + &&TARGET_BINARY_OP_SUBSCR_LIST_SLICE, + &&TARGET_BINARY_OP_SUBSCR_STR_INT, + &&TARGET_BINARY_OP_SUBSCR_TUPLE_INT, + &&TARGET_BINARY_OP_SUBSCR_USTR_INT, + &&TARGET_BINARY_OP_SUBTRACT_FLOAT, + &&TARGET_BINARY_OP_SUBTRACT_INT, + &&TARGET_CALL_ALLOC_AND_ENTER_INIT, + &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_CALL_BOUND_METHOD_GENERAL, + &&TARGET_CALL_BUILTIN_CLASS, + &&TARGET_CALL_BUILTIN_FAST, + &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, + &&TARGET_CALL_BUILTIN_O, + &&TARGET_CALL_EX_NON_PY_GENERAL, + &&TARGET_CALL_EX_PY, + &&TARGET_CALL_ISINSTANCE, + &&TARGET_CALL_KW_BOUND_METHOD, + &&TARGET_CALL_KW_NON_PY, + &&TARGET_CALL_KW_PY, + &&TARGET_CALL_LEN, + &&TARGET_CALL_LIST_APPEND, + &&TARGET_CALL_METHOD_DESCRIPTOR_FAST, + &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + &&TARGET_CALL_METHOD_DESCRIPTOR_NOARGS, + &&TARGET_CALL_METHOD_DESCRIPTOR_O, + &&TARGET_CALL_NON_PY_GENERAL, + &&TARGET_CALL_PY_EXACT_ARGS, + &&TARGET_CALL_PY_GENERAL, + &&TARGET_CALL_STR_1, + &&TARGET_CALL_TUPLE_1, + &&TARGET_CALL_TYPE_1, + &&TARGET_COMPARE_OP_FLOAT, + &&TARGET_COMPARE_OP_INT, + &&TARGET_COMPARE_OP_STR, + &&TARGET_CONTAINS_OP_DICT, + &&TARGET_CONTAINS_OP_SET, + &&TARGET_FOR_ITER_GEN, + &&TARGET_FOR_ITER_LIST, + &&TARGET_FOR_ITER_RANGE, + &&TARGET_FOR_ITER_TUPLE, + &&TARGET_FOR_ITER_VIRTUAL, + &&TARGET_GET_ITER_SELF, + &&TARGET_GET_ITER_VIRTUAL, + &&TARGET_JUMP_BACKWARD_JIT, + &&TARGET_JUMP_BACKWARD_NO_JIT, + &&TARGET_LOAD_ATTR_CLASS, + &&TARGET_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, + &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, + &&TARGET_LOAD_ATTR_INSTANCE_VALUE, + &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, + &&TARGET_LOAD_ATTR_METHOD_NO_DICT, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_LOAD_ATTR_MODULE, + &&TARGET_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, + &&TARGET_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, + &&TARGET_LOAD_ATTR_PROPERTY, + &&TARGET_LOAD_ATTR_SLOT, + &&TARGET_LOAD_ATTR_WITH_HINT, + &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_GLOBAL_MODULE, + &&TARGET_LOAD_SUPER_ATTR_ATTR, + &&TARGET_LOAD_SUPER_ATTR_METHOD, + &&TARGET_RESUME_CHECK, + &&TARGET_RESUME_CHECK_JIT, + &&TARGET_SEND_ASYNC_GEN, + &&TARGET_SEND_GEN, + &&TARGET_SEND_VIRTUAL, + &&TARGET_STORE_ATTR_INSTANCE_VALUE, + &&TARGET_STORE_ATTR_SLOT, + &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_STORE_SUBSCR_DICT, + &&TARGET_STORE_SUBSCR_LIST_INT, + &&TARGET_TO_BOOL_ALWAYS_TRUE, + &&TARGET_TO_BOOL_BOOL, + &&TARGET_TO_BOOL_INT, + &&TARGET_TO_BOOL_LIST, + &&TARGET_TO_BOOL_NONE, + &&TARGET_TO_BOOL_STR, + &&TARGET_UNPACK_SEQUENCE_LIST, + &&TARGET_UNPACK_SEQUENCE_TUPLE, + &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_INSTRUMENTED_END_FOR, + &&TARGET_INSTRUMENTED_POP_ITER, + &&TARGET_INSTRUMENTED_END_SEND, + &&TARGET_INSTRUMENTED_FOR_ITER, + &&TARGET_INSTRUMENTED_INSTRUCTION, + &&TARGET_INSTRUMENTED_JUMP_FORWARD, + &&TARGET_INSTRUMENTED_NOT_TAKEN, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_TRUE, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_NONE, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + &&TARGET_INSTRUMENTED_RESUME, + &&TARGET_INSTRUMENTED_RETURN_VALUE, + &&TARGET_INSTRUMENTED_YIELD_VALUE, + &&TARGET_INSTRUMENTED_END_ASYNC_FOR, + &&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR, + &&TARGET_INSTRUMENTED_CALL, + &&TARGET_INSTRUMENTED_CALL_KW, + &&TARGET_INSTRUMENTED_CALL_FUNCTION_EX, + &&TARGET_INSTRUMENTED_JUMP_BACKWARD, + &&TARGET_INSTRUMENTED_LINE, + &&TARGET_ENTER_EXECUTOR, + &&TARGET_TRACE_RECORD, +}; +#if _Py_TIER2 +static void *opcode_tracing_targets_table[256] = { + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, +}; +#endif +#else /* _Py_TAIL_CALL_INTERP */ +static py_tail_call_funcptr instruction_funcptr_handler_table[256]; + +static py_tail_call_funcptr instruction_funcptr_tracing_table[256]; + +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_error(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); + +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_DICT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_GETITEM(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_LIST_SLICE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_STR_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBSCR_USTR_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_INTERPOLATION(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_TEMPLATE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CACHE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_EX_NON_PY_GENERAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_EX_PY(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COPY(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_EXTENDED_OPCODE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_FOR_ITER_VIRTUAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_SELF(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_VIRTUAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_ASYNC_FOR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IS_OP(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_BORROW_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_NOP(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK_JIT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_ASYNC_GEN(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_VIRTUAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SWAP(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_TRACE_RECORD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); + +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { + int opcode = next_instr->op.code; + _PyErr_Format(tstate, PyExc_SystemError, + "%U:%d: unknown opcode %d", + _PyFrame_GetCode(frame)->co_filename, + PyUnstable_InterpreterFrame_GetLine(frame), + opcode); +JUMP_TO_LABEL(error); +} + +static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { + [BINARY_OP] = _TAIL_CALL_BINARY_OP, + [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, + [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, + [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_BINARY_OP_ADD_UNICODE, + [BINARY_OP_EXTEND] = _TAIL_CALL_BINARY_OP_EXTEND, + [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE, + [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT, + [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_BINARY_OP_MULTIPLY_INT, + [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_BINARY_OP_SUBSCR_DICT, + [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_BINARY_OP_SUBSCR_GETITEM, + [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_BINARY_OP_SUBSCR_LIST_INT, + [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_BINARY_OP_SUBSCR_LIST_SLICE, + [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_BINARY_OP_SUBSCR_STR_INT, + [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_BINARY_OP_SUBSCR_TUPLE_INT, + [BINARY_OP_SUBSCR_USTR_INT] = _TAIL_CALL_BINARY_OP_SUBSCR_USTR_INT, + [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT, + [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_BINARY_OP_SUBTRACT_INT, + [BINARY_SLICE] = _TAIL_CALL_BINARY_SLICE, + [BUILD_INTERPOLATION] = _TAIL_CALL_BUILD_INTERPOLATION, + [BUILD_LIST] = _TAIL_CALL_BUILD_LIST, + [BUILD_MAP] = _TAIL_CALL_BUILD_MAP, + [BUILD_SET] = _TAIL_CALL_BUILD_SET, + [BUILD_SLICE] = _TAIL_CALL_BUILD_SLICE, + [BUILD_STRING] = _TAIL_CALL_BUILD_STRING, + [BUILD_TEMPLATE] = _TAIL_CALL_BUILD_TEMPLATE, + [BUILD_TUPLE] = _TAIL_CALL_BUILD_TUPLE, + [CACHE] = _TAIL_CALL_CACHE, + [CALL] = _TAIL_CALL_CALL, + [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT, + [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS, + [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_CALL_BOUND_METHOD_GENERAL, + [CALL_BUILTIN_CLASS] = _TAIL_CALL_CALL_BUILTIN_CLASS, + [CALL_BUILTIN_FAST] = _TAIL_CALL_CALL_BUILTIN_FAST, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS, + [CALL_BUILTIN_O] = _TAIL_CALL_CALL_BUILTIN_O, + [CALL_EX_NON_PY_GENERAL] = _TAIL_CALL_CALL_EX_NON_PY_GENERAL, + [CALL_EX_PY] = _TAIL_CALL_CALL_EX_PY, + [CALL_FUNCTION_EX] = _TAIL_CALL_CALL_FUNCTION_EX, + [CALL_INTRINSIC_1] = _TAIL_CALL_CALL_INTRINSIC_1, + [CALL_INTRINSIC_2] = _TAIL_CALL_CALL_INTRINSIC_2, + [CALL_ISINSTANCE] = _TAIL_CALL_CALL_ISINSTANCE, + [CALL_KW] = _TAIL_CALL_CALL_KW, + [CALL_KW_BOUND_METHOD] = _TAIL_CALL_CALL_KW_BOUND_METHOD, + [CALL_KW_NON_PY] = _TAIL_CALL_CALL_KW_NON_PY, + [CALL_KW_PY] = _TAIL_CALL_CALL_KW_PY, + [CALL_LEN] = _TAIL_CALL_CALL_LEN, + [CALL_LIST_APPEND] = _TAIL_CALL_CALL_LIST_APPEND, + [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS, + [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_CALL_METHOD_DESCRIPTOR_O, + [CALL_NON_PY_GENERAL] = _TAIL_CALL_CALL_NON_PY_GENERAL, + [CALL_PY_EXACT_ARGS] = _TAIL_CALL_CALL_PY_EXACT_ARGS, + [CALL_PY_GENERAL] = _TAIL_CALL_CALL_PY_GENERAL, + [CALL_STR_1] = _TAIL_CALL_CALL_STR_1, + [CALL_TUPLE_1] = _TAIL_CALL_CALL_TUPLE_1, + [CALL_TYPE_1] = _TAIL_CALL_CALL_TYPE_1, + [CHECK_EG_MATCH] = _TAIL_CALL_CHECK_EG_MATCH, + [CHECK_EXC_MATCH] = _TAIL_CALL_CHECK_EXC_MATCH, + [CLEANUP_THROW] = _TAIL_CALL_CLEANUP_THROW, + [COMPARE_OP] = _TAIL_CALL_COMPARE_OP, + [COMPARE_OP_FLOAT] = _TAIL_CALL_COMPARE_OP_FLOAT, + [COMPARE_OP_INT] = _TAIL_CALL_COMPARE_OP_INT, + [COMPARE_OP_STR] = _TAIL_CALL_COMPARE_OP_STR, + [CONTAINS_OP] = _TAIL_CALL_CONTAINS_OP, + [CONTAINS_OP_DICT] = _TAIL_CALL_CONTAINS_OP_DICT, + [CONTAINS_OP_SET] = _TAIL_CALL_CONTAINS_OP_SET, + [CONVERT_VALUE] = _TAIL_CALL_CONVERT_VALUE, + [COPY] = _TAIL_CALL_COPY, + [COPY_FREE_VARS] = _TAIL_CALL_COPY_FREE_VARS, + [DELETE_DEREF] = _TAIL_CALL_DELETE_DEREF, + [DELETE_FAST] = _TAIL_CALL_DELETE_FAST, + [DELETE_SUBSCR] = _TAIL_CALL_DELETE_SUBSCR, + [DICT_MERGE] = _TAIL_CALL_DICT_MERGE, + [DICT_UPDATE] = _TAIL_CALL_DICT_UPDATE, + [END_ASYNC_FOR] = _TAIL_CALL_END_ASYNC_FOR, + [END_FOR] = _TAIL_CALL_END_FOR, + [END_SEND] = _TAIL_CALL_END_SEND, + [ENTER_EXECUTOR] = _TAIL_CALL_ENTER_EXECUTOR, + [EXIT_INIT_CHECK] = _TAIL_CALL_EXIT_INIT_CHECK, + [EXTENDED_ARG] = _TAIL_CALL_EXTENDED_ARG, + [EXTENDED_OPCODE] = _TAIL_CALL_EXTENDED_OPCODE, + [FORMAT_SIMPLE] = _TAIL_CALL_FORMAT_SIMPLE, + [FORMAT_WITH_SPEC] = _TAIL_CALL_FORMAT_WITH_SPEC, + [FOR_ITER] = _TAIL_CALL_FOR_ITER, + [FOR_ITER_GEN] = _TAIL_CALL_FOR_ITER_GEN, + [FOR_ITER_LIST] = _TAIL_CALL_FOR_ITER_LIST, + [FOR_ITER_RANGE] = _TAIL_CALL_FOR_ITER_RANGE, + [FOR_ITER_TUPLE] = _TAIL_CALL_FOR_ITER_TUPLE, + [FOR_ITER_VIRTUAL] = _TAIL_CALL_FOR_ITER_VIRTUAL, + [GET_AITER] = _TAIL_CALL_GET_AITER, + [GET_ANEXT] = _TAIL_CALL_GET_ANEXT, + [GET_AWAITABLE] = _TAIL_CALL_GET_AWAITABLE, + [GET_ITER] = _TAIL_CALL_GET_ITER, + [GET_ITER_SELF] = _TAIL_CALL_GET_ITER_SELF, + [GET_ITER_VIRTUAL] = _TAIL_CALL_GET_ITER_VIRTUAL, + [GET_LEN] = _TAIL_CALL_GET_LEN, + [IMPORT_FROM] = _TAIL_CALL_IMPORT_FROM, + [IMPORT_NAME] = _TAIL_CALL_IMPORT_NAME, + [INSTRUMENTED_CALL] = _TAIL_CALL_INSTRUMENTED_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_CALL_KW] = _TAIL_CALL_INSTRUMENTED_CALL_KW, + [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_INSTRUMENTED_END_ASYNC_FOR, + [INSTRUMENTED_END_FOR] = _TAIL_CALL_INSTRUMENTED_END_FOR, + [INSTRUMENTED_END_SEND] = _TAIL_CALL_INSTRUMENTED_END_SEND, + [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_INSTRUMENTED_FOR_ITER, + [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_INSTRUMENTED_INSTRUCTION, + [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD, + [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_INSTRUMENTED_JUMP_FORWARD, + [INSTRUMENTED_LINE] = _TAIL_CALL_INSTRUMENTED_LINE, + [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR, + [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_INSTRUMENTED_NOT_TAKEN, + [INSTRUMENTED_POP_ITER] = _TAIL_CALL_INSTRUMENTED_POP_ITER, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE, + [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE, + [INSTRUMENTED_RESUME] = _TAIL_CALL_INSTRUMENTED_RESUME, + [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_INSTRUMENTED_RETURN_VALUE, + [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_INSTRUMENTED_YIELD_VALUE, + [INTERPRETER_EXIT] = _TAIL_CALL_INTERPRETER_EXIT, + [IS_OP] = _TAIL_CALL_IS_OP, + [JUMP_BACKWARD] = _TAIL_CALL_JUMP_BACKWARD, + [JUMP_BACKWARD_JIT] = _TAIL_CALL_JUMP_BACKWARD_JIT, + [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT, + [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_JUMP_BACKWARD_NO_JIT, + [JUMP_FORWARD] = _TAIL_CALL_JUMP_FORWARD, + [LIST_APPEND] = _TAIL_CALL_LIST_APPEND, + [LIST_EXTEND] = _TAIL_CALL_LIST_EXTEND, + [LOAD_ATTR] = _TAIL_CALL_LOAD_ATTR, + [LOAD_ATTR_CLASS] = _TAIL_CALL_LOAD_ATTR_CLASS, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, + [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE, + [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT, + [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT, + [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES, + [LOAD_ATTR_MODULE] = _TAIL_CALL_LOAD_ATTR_MODULE, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, + [LOAD_ATTR_PROPERTY] = _TAIL_CALL_LOAD_ATTR_PROPERTY, + [LOAD_ATTR_SLOT] = _TAIL_CALL_LOAD_ATTR_SLOT, + [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_LOAD_ATTR_WITH_HINT, + [LOAD_BUILD_CLASS] = _TAIL_CALL_LOAD_BUILD_CLASS, + [LOAD_COMMON_CONSTANT] = _TAIL_CALL_LOAD_COMMON_CONSTANT, + [LOAD_CONST] = _TAIL_CALL_LOAD_CONST, + [LOAD_DEREF] = _TAIL_CALL_LOAD_DEREF, + [LOAD_FAST] = _TAIL_CALL_LOAD_FAST, + [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_LOAD_FAST_AND_CLEAR, + [LOAD_FAST_BORROW] = _TAIL_CALL_LOAD_FAST_BORROW, + [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_LOAD_FAST_BORROW_LOAD_FAST_BORROW, + [LOAD_FAST_CHECK] = _TAIL_CALL_LOAD_FAST_CHECK, + [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_LOAD_FAST_LOAD_FAST, + [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_LOAD_FROM_DICT_OR_DEREF, + [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS, + [LOAD_GLOBAL] = _TAIL_CALL_LOAD_GLOBAL, + [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_LOAD_GLOBAL_BUILTIN, + [LOAD_GLOBAL_MODULE] = _TAIL_CALL_LOAD_GLOBAL_MODULE, + [LOAD_LOCALS] = _TAIL_CALL_LOAD_LOCALS, + [LOAD_NAME] = _TAIL_CALL_LOAD_NAME, + [LOAD_SMALL_INT] = _TAIL_CALL_LOAD_SMALL_INT, + [LOAD_SPECIAL] = _TAIL_CALL_LOAD_SPECIAL, + [LOAD_SUPER_ATTR] = _TAIL_CALL_LOAD_SUPER_ATTR, + [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_LOAD_SUPER_ATTR_ATTR, + [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_LOAD_SUPER_ATTR_METHOD, + [MAKE_CELL] = _TAIL_CALL_MAKE_CELL, + [MAKE_FUNCTION] = _TAIL_CALL_MAKE_FUNCTION, + [MAP_ADD] = _TAIL_CALL_MAP_ADD, + [MATCH_CLASS] = _TAIL_CALL_MATCH_CLASS, + [MATCH_KEYS] = _TAIL_CALL_MATCH_KEYS, + [MATCH_MAPPING] = _TAIL_CALL_MATCH_MAPPING, + [MATCH_SEQUENCE] = _TAIL_CALL_MATCH_SEQUENCE, + [NOP] = _TAIL_CALL_NOP, + [NOT_TAKEN] = _TAIL_CALL_NOT_TAKEN, + [POP_EXCEPT] = _TAIL_CALL_POP_EXCEPT, + [POP_ITER] = _TAIL_CALL_POP_ITER, + [POP_JUMP_IF_FALSE] = _TAIL_CALL_POP_JUMP_IF_FALSE, + [POP_JUMP_IF_NONE] = _TAIL_CALL_POP_JUMP_IF_NONE, + [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_POP_JUMP_IF_NOT_NONE, + [POP_JUMP_IF_TRUE] = _TAIL_CALL_POP_JUMP_IF_TRUE, + [POP_TOP] = _TAIL_CALL_POP_TOP, + [PUSH_EXC_INFO] = _TAIL_CALL_PUSH_EXC_INFO, + [PUSH_NULL] = _TAIL_CALL_PUSH_NULL, + [RAISE_VARARGS] = _TAIL_CALL_RAISE_VARARGS, + [RERAISE] = _TAIL_CALL_RERAISE, + [RESERVED] = _TAIL_CALL_RESERVED, + [RESUME] = _TAIL_CALL_RESUME, + [RESUME_CHECK] = _TAIL_CALL_RESUME_CHECK, + [RESUME_CHECK_JIT] = _TAIL_CALL_RESUME_CHECK_JIT, + [RETURN_GENERATOR] = _TAIL_CALL_RETURN_GENERATOR, + [RETURN_VALUE] = _TAIL_CALL_RETURN_VALUE, + [SEND] = _TAIL_CALL_SEND, + [SEND_ASYNC_GEN] = _TAIL_CALL_SEND_ASYNC_GEN, + [SEND_GEN] = _TAIL_CALL_SEND_GEN, + [SEND_VIRTUAL] = _TAIL_CALL_SEND_VIRTUAL, + [SETUP_ANNOTATIONS] = _TAIL_CALL_SETUP_ANNOTATIONS, + [SET_ADD] = _TAIL_CALL_SET_ADD, + [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_SET_FUNCTION_ATTRIBUTE, + [SET_UPDATE] = _TAIL_CALL_SET_UPDATE, + [STORE_ATTR] = _TAIL_CALL_STORE_ATTR, + [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_STORE_ATTR_INSTANCE_VALUE, + [STORE_ATTR_SLOT] = _TAIL_CALL_STORE_ATTR_SLOT, + [STORE_ATTR_WITH_HINT] = _TAIL_CALL_STORE_ATTR_WITH_HINT, + [STORE_DEREF] = _TAIL_CALL_STORE_DEREF, + [STORE_FAST] = _TAIL_CALL_STORE_FAST, + [STORE_FAST_LOAD_FAST] = _TAIL_CALL_STORE_FAST_LOAD_FAST, + [STORE_FAST_STORE_FAST] = _TAIL_CALL_STORE_FAST_STORE_FAST, + [STORE_GLOBAL] = _TAIL_CALL_STORE_GLOBAL, + [STORE_NAME] = _TAIL_CALL_STORE_NAME, + [STORE_SLICE] = _TAIL_CALL_STORE_SLICE, + [STORE_SUBSCR] = _TAIL_CALL_STORE_SUBSCR, + [STORE_SUBSCR_DICT] = _TAIL_CALL_STORE_SUBSCR_DICT, + [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_STORE_SUBSCR_LIST_INT, + [SWAP] = _TAIL_CALL_SWAP, + [TO_BOOL] = _TAIL_CALL_TO_BOOL, + [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TO_BOOL_ALWAYS_TRUE, + [TO_BOOL_BOOL] = _TAIL_CALL_TO_BOOL_BOOL, + [TO_BOOL_INT] = _TAIL_CALL_TO_BOOL_INT, + [TO_BOOL_LIST] = _TAIL_CALL_TO_BOOL_LIST, + [TO_BOOL_NONE] = _TAIL_CALL_TO_BOOL_NONE, + [TO_BOOL_STR] = _TAIL_CALL_TO_BOOL_STR, + [TRACE_RECORD] = _TAIL_CALL_TRACE_RECORD, + [UNARY_INVERT] = _TAIL_CALL_UNARY_INVERT, + [UNARY_NEGATIVE] = _TAIL_CALL_UNARY_NEGATIVE, + [UNARY_NOT] = _TAIL_CALL_UNARY_NOT, + [UNPACK_EX] = _TAIL_CALL_UNPACK_EX, + [UNPACK_SEQUENCE] = _TAIL_CALL_UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_UNPACK_SEQUENCE_LIST, + [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TUPLE, + [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, + [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, + [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, + [117] = _TAIL_CALL_UNKNOWN_OPCODE, + [118] = _TAIL_CALL_UNKNOWN_OPCODE, + [119] = _TAIL_CALL_UNKNOWN_OPCODE, + [120] = _TAIL_CALL_UNKNOWN_OPCODE, + [121] = _TAIL_CALL_UNKNOWN_OPCODE, + [122] = _TAIL_CALL_UNKNOWN_OPCODE, + [123] = _TAIL_CALL_UNKNOWN_OPCODE, + [124] = _TAIL_CALL_UNKNOWN_OPCODE, + [125] = _TAIL_CALL_UNKNOWN_OPCODE, + [127] = _TAIL_CALL_UNKNOWN_OPCODE, + [219] = _TAIL_CALL_UNKNOWN_OPCODE, + [220] = _TAIL_CALL_UNKNOWN_OPCODE, + [221] = _TAIL_CALL_UNKNOWN_OPCODE, + [222] = _TAIL_CALL_UNKNOWN_OPCODE, + [223] = _TAIL_CALL_UNKNOWN_OPCODE, + [224] = _TAIL_CALL_UNKNOWN_OPCODE, + [225] = _TAIL_CALL_UNKNOWN_OPCODE, + [226] = _TAIL_CALL_UNKNOWN_OPCODE, + [227] = _TAIL_CALL_UNKNOWN_OPCODE, + [228] = _TAIL_CALL_UNKNOWN_OPCODE, + [229] = _TAIL_CALL_UNKNOWN_OPCODE, + [230] = _TAIL_CALL_UNKNOWN_OPCODE, + [231] = _TAIL_CALL_UNKNOWN_OPCODE, + [232] = _TAIL_CALL_UNKNOWN_OPCODE, +}; +static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { + [BINARY_OP] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_EXTEND] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_USTR_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_SLICE] = _TAIL_CALL_TRACE_RECORD, + [BUILD_INTERPOLATION] = _TAIL_CALL_TRACE_RECORD, + [BUILD_LIST] = _TAIL_CALL_TRACE_RECORD, + [BUILD_MAP] = _TAIL_CALL_TRACE_RECORD, + [BUILD_SET] = _TAIL_CALL_TRACE_RECORD, + [BUILD_SLICE] = _TAIL_CALL_TRACE_RECORD, + [BUILD_STRING] = _TAIL_CALL_TRACE_RECORD, + [BUILD_TEMPLATE] = _TAIL_CALL_TRACE_RECORD, + [BUILD_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [CACHE] = _TAIL_CALL_TRACE_RECORD, + [CALL] = _TAIL_CALL_TRACE_RECORD, + [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_TRACE_RECORD, + [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_CLASS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_FAST] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_O] = _TAIL_CALL_TRACE_RECORD, + [CALL_EX_NON_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_EX_PY] = _TAIL_CALL_TRACE_RECORD, + [CALL_FUNCTION_EX] = _TAIL_CALL_TRACE_RECORD, + [CALL_INTRINSIC_1] = _TAIL_CALL_TRACE_RECORD, + [CALL_INTRINSIC_2] = _TAIL_CALL_TRACE_RECORD, + [CALL_ISINSTANCE] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_BOUND_METHOD] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_NON_PY] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_PY] = _TAIL_CALL_TRACE_RECORD, + [CALL_LEN] = _TAIL_CALL_TRACE_RECORD, + [CALL_LIST_APPEND] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_TRACE_RECORD, + [CALL_NON_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_PY_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_STR_1] = _TAIL_CALL_TRACE_RECORD, + [CALL_TUPLE_1] = _TAIL_CALL_TRACE_RECORD, + [CALL_TYPE_1] = _TAIL_CALL_TRACE_RECORD, + [CHECK_EG_MATCH] = _TAIL_CALL_TRACE_RECORD, + [CHECK_EXC_MATCH] = _TAIL_CALL_TRACE_RECORD, + [CLEANUP_THROW] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP_INT] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP_STR] = _TAIL_CALL_TRACE_RECORD, + [CONTAINS_OP] = _TAIL_CALL_TRACE_RECORD, + [CONTAINS_OP_DICT] = _TAIL_CALL_TRACE_RECORD, + [CONTAINS_OP_SET] = _TAIL_CALL_TRACE_RECORD, + [CONVERT_VALUE] = _TAIL_CALL_TRACE_RECORD, + [COPY] = _TAIL_CALL_TRACE_RECORD, + [COPY_FREE_VARS] = _TAIL_CALL_TRACE_RECORD, + [DELETE_DEREF] = _TAIL_CALL_TRACE_RECORD, + [DELETE_FAST] = _TAIL_CALL_TRACE_RECORD, + [DELETE_SUBSCR] = _TAIL_CALL_TRACE_RECORD, + [DICT_MERGE] = _TAIL_CALL_TRACE_RECORD, + [DICT_UPDATE] = _TAIL_CALL_TRACE_RECORD, + [END_ASYNC_FOR] = _TAIL_CALL_TRACE_RECORD, + [END_FOR] = _TAIL_CALL_TRACE_RECORD, + [END_SEND] = _TAIL_CALL_TRACE_RECORD, + [ENTER_EXECUTOR] = _TAIL_CALL_TRACE_RECORD, + [EXIT_INIT_CHECK] = _TAIL_CALL_TRACE_RECORD, + [EXTENDED_ARG] = _TAIL_CALL_TRACE_RECORD, + [EXTENDED_OPCODE] = _TAIL_CALL_TRACE_RECORD, + [FORMAT_SIMPLE] = _TAIL_CALL_TRACE_RECORD, + [FORMAT_WITH_SPEC] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_GEN] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_LIST] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_RANGE] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_VIRTUAL] = _TAIL_CALL_TRACE_RECORD, + [GET_AITER] = _TAIL_CALL_TRACE_RECORD, + [GET_ANEXT] = _TAIL_CALL_TRACE_RECORD, + [GET_AWAITABLE] = _TAIL_CALL_TRACE_RECORD, + [GET_ITER] = _TAIL_CALL_TRACE_RECORD, + [GET_ITER_SELF] = _TAIL_CALL_TRACE_RECORD, + [GET_ITER_VIRTUAL] = _TAIL_CALL_TRACE_RECORD, + [GET_LEN] = _TAIL_CALL_TRACE_RECORD, + [IMPORT_FROM] = _TAIL_CALL_TRACE_RECORD, + [IMPORT_NAME] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_CALL] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_CALL_KW] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_END_FOR] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_END_SEND] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_LINE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_ITER] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_RESUME] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, + [INTERPRETER_EXIT] = _TAIL_CALL_TRACE_RECORD, + [IS_OP] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD_JIT] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_TRACE_RECORD, + [JUMP_FORWARD] = _TAIL_CALL_TRACE_RECORD, + [LIST_APPEND] = _TAIL_CALL_TRACE_RECORD, + [LIST_EXTEND] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_CLASS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_MODULE] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_PROPERTY] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_SLOT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_BUILD_CLASS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_COMMON_CONSTANT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_CONST] = _TAIL_CALL_TRACE_RECORD, + [LOAD_DEREF] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_BORROW] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_CHECK] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_GLOBAL] = _TAIL_CALL_TRACE_RECORD, + [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_TRACE_RECORD, + [LOAD_GLOBAL_MODULE] = _TAIL_CALL_TRACE_RECORD, + [LOAD_LOCALS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_NAME] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SMALL_INT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SPECIAL] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SUPER_ATTR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_TRACE_RECORD, + [MAKE_CELL] = _TAIL_CALL_TRACE_RECORD, + [MAKE_FUNCTION] = _TAIL_CALL_TRACE_RECORD, + [MAP_ADD] = _TAIL_CALL_TRACE_RECORD, + [MATCH_CLASS] = _TAIL_CALL_TRACE_RECORD, + [MATCH_KEYS] = _TAIL_CALL_TRACE_RECORD, + [MATCH_MAPPING] = _TAIL_CALL_TRACE_RECORD, + [MATCH_SEQUENCE] = _TAIL_CALL_TRACE_RECORD, + [NOP] = _TAIL_CALL_TRACE_RECORD, + [NOT_TAKEN] = _TAIL_CALL_TRACE_RECORD, + [POP_EXCEPT] = _TAIL_CALL_TRACE_RECORD, + [POP_ITER] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_FALSE] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_NONE] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_TRUE] = _TAIL_CALL_TRACE_RECORD, + [POP_TOP] = _TAIL_CALL_TRACE_RECORD, + [PUSH_EXC_INFO] = _TAIL_CALL_TRACE_RECORD, + [PUSH_NULL] = _TAIL_CALL_TRACE_RECORD, + [RAISE_VARARGS] = _TAIL_CALL_TRACE_RECORD, + [RERAISE] = _TAIL_CALL_TRACE_RECORD, + [RESERVED] = _TAIL_CALL_TRACE_RECORD, + [RESUME] = _TAIL_CALL_TRACE_RECORD, + [RESUME_CHECK] = _TAIL_CALL_TRACE_RECORD, + [RESUME_CHECK_JIT] = _TAIL_CALL_TRACE_RECORD, + [RETURN_GENERATOR] = _TAIL_CALL_TRACE_RECORD, + [RETURN_VALUE] = _TAIL_CALL_TRACE_RECORD, + [SEND] = _TAIL_CALL_TRACE_RECORD, + [SEND_ASYNC_GEN] = _TAIL_CALL_TRACE_RECORD, + [SEND_GEN] = _TAIL_CALL_TRACE_RECORD, + [SEND_VIRTUAL] = _TAIL_CALL_TRACE_RECORD, + [SETUP_ANNOTATIONS] = _TAIL_CALL_TRACE_RECORD, + [SET_ADD] = _TAIL_CALL_TRACE_RECORD, + [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_TRACE_RECORD, + [SET_UPDATE] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR_SLOT] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR_WITH_HINT] = _TAIL_CALL_TRACE_RECORD, + [STORE_DEREF] = _TAIL_CALL_TRACE_RECORD, + [STORE_FAST] = _TAIL_CALL_TRACE_RECORD, + [STORE_FAST_LOAD_FAST] = _TAIL_CALL_TRACE_RECORD, + [STORE_FAST_STORE_FAST] = _TAIL_CALL_TRACE_RECORD, + [STORE_GLOBAL] = _TAIL_CALL_TRACE_RECORD, + [STORE_NAME] = _TAIL_CALL_TRACE_RECORD, + [STORE_SLICE] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR_DICT] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_TRACE_RECORD, + [SWAP] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_BOOL] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_INT] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_LIST] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_NONE] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_STR] = _TAIL_CALL_TRACE_RECORD, + [TRACE_RECORD] = _TAIL_CALL_TRACE_RECORD, + [UNARY_INVERT] = _TAIL_CALL_TRACE_RECORD, + [UNARY_NEGATIVE] = _TAIL_CALL_TRACE_RECORD, + [UNARY_NOT] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_EX] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [WITH_EXCEPT_START] = _TAIL_CALL_TRACE_RECORD, + [YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, + [117] = _TAIL_CALL_UNKNOWN_OPCODE, + [118] = _TAIL_CALL_UNKNOWN_OPCODE, + [119] = _TAIL_CALL_UNKNOWN_OPCODE, + [120] = _TAIL_CALL_UNKNOWN_OPCODE, + [121] = _TAIL_CALL_UNKNOWN_OPCODE, + [122] = _TAIL_CALL_UNKNOWN_OPCODE, + [123] = _TAIL_CALL_UNKNOWN_OPCODE, + [124] = _TAIL_CALL_UNKNOWN_OPCODE, + [125] = _TAIL_CALL_UNKNOWN_OPCODE, + [127] = _TAIL_CALL_UNKNOWN_OPCODE, + [219] = _TAIL_CALL_UNKNOWN_OPCODE, + [220] = _TAIL_CALL_UNKNOWN_OPCODE, + [221] = _TAIL_CALL_UNKNOWN_OPCODE, + [222] = _TAIL_CALL_UNKNOWN_OPCODE, + [223] = _TAIL_CALL_UNKNOWN_OPCODE, + [224] = _TAIL_CALL_UNKNOWN_OPCODE, + [225] = _TAIL_CALL_UNKNOWN_OPCODE, + [226] = _TAIL_CALL_UNKNOWN_OPCODE, + [227] = _TAIL_CALL_UNKNOWN_OPCODE, + [228] = _TAIL_CALL_UNKNOWN_OPCODE, + [229] = _TAIL_CALL_UNKNOWN_OPCODE, + [230] = _TAIL_CALL_UNKNOWN_OPCODE, + [231] = _TAIL_CALL_UNKNOWN_OPCODE, + [232] = _TAIL_CALL_UNKNOWN_OPCODE, +}; +#endif /* _Py_TAIL_CALL_INTERP */ diff --git a/cinderx/Interpreter/3.16/interpreter.c b/cinderx/Interpreter/3.16/interpreter.c new file mode 100644 index 000000000..770f6bca4 --- /dev/null +++ b/cinderx/Interpreter/3.16/interpreter.c @@ -0,0 +1,935 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +// clang-format off + +#define CINDERX_INTERPRETER + +#include "cinderx/UpstreamBorrow/borrowed.h" +#include "cinderx/Interpreter/cinder_opcode.h" + +#include "cinderx/module_c_state.h" + +#include "cinderx/Common/code.h" + +//#include "internal/pycore_opcode.h" + +// Must come after pycore_opcode, we want to get the exported ones. +#define NEED_OPCODE_NAMES +#define NEED_OPCODE_TABLES +#include "cinderx/Interpreter/cinder_opcode.h" + +#include "internal/pycore_ceval.h" +#include "internal/pycore_stackref.h" +#include "internal/pycore_interpframe.h" + +#include "cinderx/StaticPython/classloader.h" +#include "cinderx/StaticPython/checked_dict.h" +#include "cinderx/StaticPython/checked_list.h" +#include "cinderx/StaticPython/static_array.h" + +#include "cinderx/Jit/generators_rt.h" + +#undef EXTRA_CASES + +#define EXTRA_CASES \ + case 120: \ + case 122: \ + case 123: \ + case 124: \ + case 125: \ + case 127: \ + case 214: \ + case 215: \ + case 216: \ + case 217: \ + case 218: \ + case 219: \ + case 220: \ + case 221: \ + case 222: \ + case 223: \ + case 224: \ + case 225: \ + case 226: \ + case 227: \ + case 228: \ + case 229: \ + case 230: \ + case 231: \ + case 232: \ + ; + +#ifdef ENABLE_INTERPRETER_LOOP + +bool is_adaptive_enabled(CodeExtra *extra) { + return !Ci_GetDelayAdaptiveCode() || Ci_code_extra_get_calls(extra) > Ci_GetAdaptiveThreshold(); +} + +#endif + +/* _PyEval_EvalFrameDefault() is a *big* function, + * so consume 3 units of C stack */ +#define PY_EVAL_C_STACK_UNITS 2 + +// These are used to truncate primitives/check signed bits when converting +// between them + + +#ifdef ENABLE_INTERPRETER_LOOP + +static uint64_t trunc_masks[] = {0xFF, 0xFFFF, 0xFFFFFFFF, 0xFFFFFFFFFFFFFFFF}; +static uint64_t signed_bits[] = {0x80, 0x8000, 0x80000000, 0x8000000000000000}; +static uint64_t signex_masks[] = { + 0xFFFFFFFFFFFFFF00, + 0xFFFFFFFFFFFF0000, + 0xFFFFFFFF00000000, + 0x0}; + +static int8_t unbox_primitive_bool(PyObject* x) { + assert(PyBool_Check(x)); + return (x == Py_True) ? 1 : 0; +} + +static Py_ssize_t unbox_primitive_int(PyObject* x) { + assert(PyLong_Check(x)); + return (Py_ssize_t)PyLong_AsVoidPtr(x); +} + +static PyObject* box_primitive(int type, Py_ssize_t value) { + switch (type) { + case TYPED_BOOL: + return PyBool_FromLong((int8_t)value); + case TYPED_INT8: + case TYPED_CHAR: + return PyLong_FromSsize_t((int8_t)value); + case TYPED_INT16: + return PyLong_FromSsize_t((int16_t)value); + case TYPED_INT32: + return PyLong_FromSsize_t((int32_t)value); + case TYPED_INT64: + return PyLong_FromSsize_t((int64_t)value); + case TYPED_UINT8: + return PyLong_FromSize_t((uint8_t)value); + case TYPED_UINT16: + return PyLong_FromSize_t((uint16_t)value); + case TYPED_UINT32: + return PyLong_FromSize_t((uint32_t)value); + case TYPED_UINT64: + return PyLong_FromSize_t((uint64_t)value); + default: + assert(0); + return NULL; + } +} + +static _PyStackRef sign_extend_primitive(_PyStackRef obj, int type) { + if ((type & (TYPED_INT_SIGNED)) && type != (TYPED_DOUBLE)) { + /* We have a boxed value on the stack already, but we may have to + * deal with sign extension */ + PyObject* val = PyStackRef_AsPyObjectBorrow(obj); + size_t ival = (size_t)PyLong_AsVoidPtr(val); + if (ival & ((size_t)1) << 63) { + PyStackRef_CLOSE(obj); + return PyStackRef_FromPyObjectSteal(PyLong_FromSsize_t((int64_t)ival)); + } + } + return obj; +} + +static PyObject* load_field(int field_type, void* addr) { + PyObject* value; + switch (field_type) { + case TYPED_BOOL: + value = PyBool_FromLong(*(int8_t*)addr); + break; + case TYPED_INT8: + value = PyLong_FromVoidPtr((void*)(Py_ssize_t) * ((int8_t*)addr)); + break; + case TYPED_INT16: + value = PyLong_FromVoidPtr((void*)(Py_ssize_t) * ((int16_t*)addr)); + break; + case TYPED_INT32: + value = PyLong_FromVoidPtr((void*)(Py_ssize_t) * ((int32_t*)addr)); + break; + case TYPED_INT64: + value = PyLong_FromVoidPtr((void*)(Py_ssize_t) * ((int64_t*)addr)); + break; + case TYPED_UINT8: + value = PyLong_FromVoidPtr((void*)(Py_ssize_t) * ((uint8_t*)addr)); + break; + case TYPED_UINT16: + value = PyLong_FromVoidPtr((void*)(Py_ssize_t) * ((uint16_t*)addr)); + break; + case TYPED_UINT32: + value = PyLong_FromVoidPtr((void*)(Py_ssize_t) * ((uint32_t*)addr)); + break; + case TYPED_UINT64: + value = PyLong_FromVoidPtr((void*)(Py_ssize_t) * ((uint64_t*)addr)); + break; + case TYPED_DOUBLE: + value = PyFloat_FromDouble(*(double*)addr); + break; + default: + PyErr_SetString(PyExc_RuntimeError, "unsupported field type"); + return NULL; + } + return value; +} + +static void store_field(int field_type, void* addr, PyObject* value) { + switch (field_type) { + case TYPED_BOOL: + *(int8_t*)addr = (int8_t)unbox_primitive_bool(value); + break; + case TYPED_INT8: + *(int8_t*)addr = (int8_t)unbox_primitive_int(value); + break; + case TYPED_INT16: + *(int16_t*)addr = (int16_t)unbox_primitive_int(value); + break; + case TYPED_INT32: + *(int32_t*)addr = (int32_t)unbox_primitive_int(value); + break; + case TYPED_INT64: + *(int64_t*)addr = (int64_t)unbox_primitive_int(value); + break; + case TYPED_UINT8: + *(uint8_t*)addr = (uint8_t)unbox_primitive_int(value); + break; + case TYPED_UINT16: + *(uint16_t*)addr = (uint16_t)unbox_primitive_int(value); + break; + case TYPED_UINT32: + *(uint32_t*)addr = (uint32_t)unbox_primitive_int(value); + break; + case TYPED_UINT64: + *(uint64_t*)addr = (uint64_t)unbox_primitive_int(value); + break; + case TYPED_DOUBLE: + *((double*)addr) = PyFloat_AsDouble(value); + break; + default: + PyErr_SetString(PyExc_RuntimeError, "unsupported field type"); + break; + } +} + +#define FIELD_OFFSET(self, offset) (PyObject**)(((char*)self) + offset) + +static int ci_build_dict(_PyStackRef *map_items, Py_ssize_t map_size, PyObject *map) +{ + for (Py_ssize_t i = 0; i < map_size; i++) { + PyObject* key = PyStackRef_AsPyObjectBorrow(map_items[2 * i]); + PyObject* value = PyStackRef_AsPyObjectBorrow(map_items[2 * i + 1]); + if (Ci_CheckedDict_SetItem(map, key, value) < 0) { + return -1; + } + } + return 0; +} + +#if ENABLE_SPECIALIZATION && defined(ENABLE_ADAPTIVE_STATIC_PYTHON) +static void specialize_with_value(_Py_CODEUNIT next_instr, PyObject *value, int opcode, + int shift, int bits) +{ + int32_t index = _PyClassLoader_CacheValue(value); + if (index >= 0 && index <= (INT32_MAX >> 2)) { + int32_t *cache = (int32_t*)next_instr; + *cache = (int32_t)(index << shift) | bits; + _Ci_specialize(next_instr, opcode); + } +} +#endif + +#define INT_UNARY_OPCODE(opid, op) \ + case opid: \ + res = PyLong_FromVoidPtr((void*)(op(size_t) PyLong_AsVoidPtr(val))); \ + break; + +#define DBL_UNARY_OPCODE(opid, op) \ + case opid: \ + res = PyFloat_FromDouble(op(PyFloat_AS_DOUBLE(val))); \ + break; + +static PyObject * +primitive_unary_op(PyObject *val, int oparg) +{ + PyObject *res; + switch (oparg) { + INT_UNARY_OPCODE(PRIM_OP_NEG_INT, -) + INT_UNARY_OPCODE(PRIM_OP_INV_INT, ~) + DBL_UNARY_OPCODE(PRIM_OP_NEG_DBL, -) + case PRIM_OP_NOT_INT: { + res = PyLong_AsVoidPtr(val) ? Py_False : Py_True; + Py_INCREF(res); + break; + } + default: + PyErr_SetString(PyExc_RuntimeError, "unknown op"); + return NULL; + } + return res; +} + +#define INT_BIN_OPCODE_UNSIGNED(opid, op) \ + case opid: \ + res = PyLong_FromVoidPtr((void*)(((size_t)PyLong_AsVoidPtr(l))op( \ + (size_t)PyLong_AsVoidPtr(r)))); \ + break; + +#define INT_BIN_OPCODE_SIGNED(opid, op) \ + case opid: \ + res = PyLong_FromVoidPtr((void*)(((Py_ssize_t)PyLong_AsVoidPtr(l))op( \ + (Py_ssize_t)PyLong_AsVoidPtr(r)))); \ + break; + +#define DOUBLE_BIN_OPCODE(opid, op) \ + case opid: \ + res = (PyFloat_FromDouble((PyFloat_AS_DOUBLE(l))op(PyFloat_AS_DOUBLE(r)))); \ + break; + +static PyObject * +primitive_binary_op(PyObject *l, PyObject *r, int oparg) +{ + PyObject *res; + switch (oparg) { + INT_BIN_OPCODE_UNSIGNED(PRIM_OP_ADD_INT, +) + INT_BIN_OPCODE_UNSIGNED(PRIM_OP_SUB_INT, -) + INT_BIN_OPCODE_UNSIGNED(PRIM_OP_MUL_INT, *) + INT_BIN_OPCODE_SIGNED(PRIM_OP_DIV_INT, /) + INT_BIN_OPCODE_SIGNED(PRIM_OP_MOD_INT, %) + case PRIM_OP_POW_INT: { + double power = + pow((Py_ssize_t)PyLong_AsVoidPtr(l), + (Py_ssize_t)PyLong_AsVoidPtr(r)); + res = PyFloat_FromDouble(power); + break; + } + case PRIM_OP_POW_UN_INT: { + double power = + pow((size_t)PyLong_AsVoidPtr(l), (size_t)PyLong_AsVoidPtr(r)); + res = PyFloat_FromDouble(power); + break; + } + + INT_BIN_OPCODE_SIGNED(PRIM_OP_LSHIFT_INT, <<) + INT_BIN_OPCODE_SIGNED(PRIM_OP_RSHIFT_INT, >>) + INT_BIN_OPCODE_UNSIGNED(PRIM_OP_XOR_INT, ^) + INT_BIN_OPCODE_UNSIGNED(PRIM_OP_OR_INT, |) + INT_BIN_OPCODE_UNSIGNED(PRIM_OP_AND_INT, &) + INT_BIN_OPCODE_UNSIGNED(PRIM_OP_MOD_UN_INT, %) + INT_BIN_OPCODE_UNSIGNED(PRIM_OP_DIV_UN_INT, /) + INT_BIN_OPCODE_UNSIGNED(PRIM_OP_RSHIFT_UN_INT, >>) + DOUBLE_BIN_OPCODE(PRIM_OP_ADD_DBL, +) + DOUBLE_BIN_OPCODE(PRIM_OP_SUB_DBL, -) + DOUBLE_BIN_OPCODE(PRIM_OP_MUL_DBL, *) + DOUBLE_BIN_OPCODE(PRIM_OP_DIV_DBL, /) + case PRIM_OP_POW_DBL: { + double power = pow(PyFloat_AsDouble(l), PyFloat_AsDouble(r)); + res = PyFloat_FromDouble(power); + break; + } + default: + PyErr_SetString(PyExc_RuntimeError, "unknown op"); + return NULL; + } + return res; +} + +#define INT_CMP_OPCODE_UNSIGNED(opid, op) \ + case opid: \ + right = (size_t)PyLong_AsVoidPtr(r); \ + left = (size_t)PyLong_AsVoidPtr(l); \ + res = (left op right) ? Py_True : Py_False; \ + Py_INCREF(res); \ + break; + +#define INT_CMP_OPCODE_SIGNED(opid, op) \ + case opid: \ + sright = (Py_ssize_t)PyLong_AsVoidPtr(r); \ + sleft = (Py_ssize_t)PyLong_AsVoidPtr(l); \ + res = (sleft op sright) ? Py_True : Py_False; \ + Py_INCREF(res); \ + break; + +#define DBL_CMP_OPCODE(opid, op) \ + case opid: \ + res = \ + ((PyFloat_AS_DOUBLE(l) op PyFloat_AS_DOUBLE(r)) ? Py_True : Py_False); \ + Py_INCREF(res); \ + break; + +static PyObject * +primitive_compare_op(PyObject *l, PyObject *r, int oparg) +{ + PyObject *res; + Py_ssize_t sleft, sright; + size_t left, right; + switch (oparg) { + INT_CMP_OPCODE_SIGNED(PRIM_OP_EQ_INT, ==) + INT_CMP_OPCODE_SIGNED(PRIM_OP_NE_INT, !=) + INT_CMP_OPCODE_SIGNED(PRIM_OP_LT_INT, <) + INT_CMP_OPCODE_SIGNED(PRIM_OP_GT_INT, >) + INT_CMP_OPCODE_SIGNED(PRIM_OP_LE_INT, <=) + INT_CMP_OPCODE_SIGNED(PRIM_OP_GE_INT, >=) + INT_CMP_OPCODE_UNSIGNED(PRIM_OP_LT_UN_INT, <) + INT_CMP_OPCODE_UNSIGNED(PRIM_OP_GT_UN_INT, >) + INT_CMP_OPCODE_UNSIGNED(PRIM_OP_LE_UN_INT, <=) + INT_CMP_OPCODE_UNSIGNED(PRIM_OP_GE_UN_INT, >=) + DBL_CMP_OPCODE(PRIM_OP_EQ_DBL, ==) + DBL_CMP_OPCODE(PRIM_OP_NE_DBL, !=) + DBL_CMP_OPCODE(PRIM_OP_LT_DBL, <) + DBL_CMP_OPCODE(PRIM_OP_GT_DBL, >) + DBL_CMP_OPCODE(PRIM_OP_LE_DBL, <=) + DBL_CMP_OPCODE(PRIM_OP_GE_DBL, >=) + default: + PyErr_SetString(PyExc_RuntimeError, "unknown op"); + return NULL; + } + return res; +} + +#define INVOKE_FUNCTION_CACHE_SIZE 4 +#define TP_ALLOC_CACHE_SIZE 2 +#define STORE_LOCAL_CACHE_SIZE 1 +#define INLINE_CACHE_ENTRIES_LOAD_FIELD 2 +#define INLINE_CACHE_ENTRIES_STORE_FIELD 2 +#define CAST_CACHE_SIZE 2 +#define INLINE_CACHE_ENTRIES_BUILD_CHECKED_LIST 2 +#define INLINE_CACHE_ENTRIES_BUILD_CHECKED_MAP 2 +#endif + +#ifdef ENABLE_INTERPRETER_LOOP + +PyObject* _Py_HOT_FUNCTION +Ci_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag); + +#include "cinderx/Interpreter/3.16/ceval.h" +#include "Python/ceval.h" +#include "cinderx/Interpreter/3.16/Includes/ceval_macros.h" + +#endif + +#define _PyEval_GetAwaitable Ci_PyEval_GetAwaitable +#define _PyEval_GetANext Ci_PyEval_GetANext + +void Ci_InitOpcodes() { +#ifdef ENABLE_ADAPTIVE_STATIC_PYTHON + // patch CPython's opcode data + for (int i = 0; i < sizeof(_CiOpcode_Caches) / sizeof(_CiOpcode_Caches[0]); i++) { + _PyOpcode_Caches[i] = _CiOpcode_Caches[i]; + } + for (int i = 0; i < sizeof(_CiOpcode_Deopt) / sizeof(_CiOpcode_Deopt[0]); i++) { + _PyOpcode_Deopt[i] = _CiOpcode_Deopt[i]; + } +#endif +} + +static void +_Ci_specialize(_Py_CODEUNIT *next_instr, int opcode) +{ + (next_instr - 1)->op.code = opcode; +} + +int load_method_static_cached_oparg(Py_ssize_t slot, bool is_classmethod) { + return (slot << 1) | (is_classmethod ? 1 : 0); +} + +bool load_method_static_cached_oparg_is_classmethod(int oparg) { + return (oparg & 1) != 0; +} + +Py_ssize_t load_method_static_cached_oparg_slot(int oparg) { + return oparg >> 1; +} + +#if defined(__GNUC__) || defined(__clang__) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wunused-label" +#elif defined(_MSC_VER) /* MS_WINDOWS */ +# pragma warning(push) +# pragma warning(disable:4102) +#endif + +#ifdef Py_DEBUG +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_assert_within_stack_bounds(frame, stack_pointer, (F), (L)) +#else +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0 +#endif + + +// CO_NO_MONITORING_EVENTS indicates the code object is read-only and therefore +// cannot have code-extra data added. +#define CI_SET_ADAPTIVE_INTERPRETER_ENABLED_STATE \ + do { \ + PyObject* executable = PyStackRef_AsPyObjectBorrow(frame->f_executable); \ + if (PyCode_Check(executable)) { \ + PyCodeObject* code = (PyCodeObject*)executable; \ + if (!(code->co_flags & CO_NO_MONITORING_EVENTS)) { \ + CodeExtra* extra = codeExtra(code); \ + adaptive_enabled = extra != NULL && is_adaptive_enabled(extra); \ + } \ + } \ + } while (0); + +#define CI_UPDATE_CALL_COUNT \ + do { \ + PyObject* executable = PyStackRef_AsPyObjectBorrow(frame->f_executable); \ + if (PyCode_Check(executable)) { \ + PyCodeObject* code = (PyCodeObject*)executable; \ + if (!(code->co_flags & CO_NO_MONITORING_EVENTS)) { \ + CodeExtra* extra = codeExtra(code); \ + if (extra == NULL) { \ + adaptive_enabled = false; \ + } else { \ + Ci_code_extra_incr_calls(extra); \ + adaptive_enabled = is_adaptive_enabled(extra); \ + } \ + } \ + } \ + } while (0); + + #undef DISPATCH_INLINED + + #define DISPATCH_INLINED(NEW_FRAME) \ + do { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + _PyFrame_StackPointerValidate(frame); \ + assert((NEW_FRAME)->previous == frame); \ + frame = tstate->current_frame = (NEW_FRAME); \ + CALL_STAT_INC(inlined_py_calls); \ + JUMP_TO_LABEL(start_frame); \ + } while (0) + +#undef IS_PEP523_HOOKED + +#define IS_PEP523_HOOKED(tstate) \ + (tstate->interp->eval_frame != NULL && \ + tstate->interp->eval_frame != Ci_EvalFrame) + +#ifdef ENABLE_INTERPRETER_LOOP + +#if _Py_TAIL_CALL_INTERP +#include "cinderx/Interpreter/cinderx_opcode_targets.h" +#include "cinderx/Interpreter/3.16/Includes/generated_cases.c.h" +#endif + +PyObject* _Py_HOT_FUNCTION +Ci_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) +{ +#if USE_COMPUTED_GOTOS && !_Py_TAIL_CALL_INTERP +/* Import the static jump table */ +#include "cinderx/Interpreter/cinderx_opcode_targets.h" +void **opcode_targets = opcode_targets_table; +#endif + +#ifdef Py_STATS + int lastopcode = 0; +#endif +#if !_Py_TAIL_CALL_INTERP + uint8_t opcode; /* Current opcode */ + int oparg; /* Current opcode argument, if any */ + assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL); +#endif + _PyEntryFrame entry; + + if (_Py_EnterRecursiveCallTstate(tstate, "")) { + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyEval_FrameClearAndPop(tstate, frame); + return NULL; + } + + /* Local "register" variables. + * These are cached values from the frame and code object. */ + _Py_CODEUNIT *next_instr; + _PyStackRef *stack_pointer; + entry.stack[0] = PyStackRef_NULL; +#ifdef Py_STACKREF_DEBUG + entry.frame.f_funcobj = PyStackRef_None; +#elif defined(Py_DEBUG) + /* Set these to invalid but identifiable values for debugging. */ + entry.frame.f_funcobj = (_PyStackRef){.bits = 0xaaa0}; + entry.frame.f_locals = (PyObject*)0xaaa1; + entry.frame.frame_obj = (PyFrameObject*)0xaaa2; + entry.frame.f_globals = (PyObject*)0xaaa3; + entry.frame.f_builtins = (PyObject*)0xaaa4; +#endif + entry.frame.f_executable = PyStackRef_None; + entry.frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR + 1; + entry.frame.stackpointer = entry.stack; + entry.frame.owner = FRAME_OWNED_BY_INTERPRETER; + entry.frame.visited = 0; + entry.frame.return_offset = 0; +#ifdef Py_DEBUG + entry.frame.lltrace = 0; + entry.frame.stackpointer_valid = 1; +#endif + /* Push frame */ + entry.frame.previous = tstate->current_frame; + frame->previous = &entry.frame; + tstate->current_frame = frame; + entry.frame.localsplus[0] = PyStackRef_NULL; +#ifdef _Py_TIER2 + if (tstate->current_executor != NULL) { + entry.frame.localsplus[0] = PyStackRef_FromPyObjectNew(tstate->current_executor); + tstate->current_executor = NULL; + } +#endif + + bool adaptive_enabled = false; + + // Suppress unused variable warning because it's too hard to improve the + // variable's scope to avoid an unused-but-set-variable warning. + (void)adaptive_enabled; + + /* support for generator.throw() */ + if (throwflag) { + if (_Py_EnterRecursivePy(tstate)) { + goto early_exit; + } +#ifdef Py_GIL_DISABLED + /* Load thread-local bytecode */ + if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + if (bytecode == NULL) { + goto early_exit; + } + ptrdiff_t off = frame->instr_ptr - _PyFrame_GetBytecode(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + } +#endif + /* Because this avoids the RESUME, we need to update instrumentation */ + _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + next_instr = frame->instr_ptr; + monitor_throw(tstate, frame, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); +#if _Py_TAIL_CALL_INTERP +# if Py_STATS + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0, lastopcode, adaptive_enabled); +# else + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0, adaptive_enabled); +# endif +#else + goto error; +#endif + } + + #if defined(_Py_TIER2) && !defined(_Py_JIT) + /* Tier 2 interpreter state */ + _PyExecutorObject *current_executor = NULL; + const _PyUOpInstruction *next_uop = NULL; +#endif +#if _Py_TAIL_CALL_INTERP +# if Py_STATS + return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_handler_table, 0, lastopcode, adaptive_enabled); +# else + return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_handler_table, 0, adaptive_enabled); +# endif +#else + goto start_frame; +#include "cinderx/Interpreter/3.16/Includes/generated_cases.c.h" +#endif + + +#ifdef _Py_TIER2 + +// Tier 2 is also here! +enter_tier_two: + +#ifdef _Py_JIT + assert(0); +#else + +#undef LOAD_IP +#define LOAD_IP(UNUSED) (void)0 + +#ifdef Py_STATS +// Disable these macros that apply to Tier 1 stats when we are in Tier 2 +#undef STAT_INC +#define STAT_INC(opname, name) ((void)0) +#undef STAT_DEC +#define STAT_DEC(opname, name) ((void)0) +#endif + +#undef ENABLE_SPECIALIZATION +#define ENABLE_SPECIALIZATION 0 +#undef ENABLE_SPECIALIZATION_FT +#define ENABLE_SPECIALIZATION_FT 0 + + ; // dummy statement after a label, before a declaration + uint16_t uopcode; +#ifdef Py_STATS + int lastuop = 0; + uint64_t trace_uop_execution_counter = 0; +#endif + + assert(next_uop->opcode == _START_EXECUTOR); +tier2_dispatch: + for (;;) { + uopcode = next_uop->opcode; +#ifdef Py_DEBUG + if (frame->lltrace >= 3) { + dump_stack(frame, stack_pointer); + if (next_uop->opcode == _START_EXECUTOR) { + printf("%4d uop: ", 0); + } + else { + printf("%4d uop: ", (int)(next_uop - current_executor->trace)); + } + _PyUOpPrint(next_uop); + printf("\n"); + } +#endif + next_uop++; + OPT_STAT_INC(uops_executed); + UOP_STAT_INC(uopcode, execution_count); + UOP_PAIR_INC(uopcode, lastuop); +#ifdef Py_STATS + trace_uop_execution_counter++; + ((_PyUOpInstruction *)next_uop)[-1].execution_count++; +#endif + + switch (uopcode) { + +#include "executor_cases.c.h" + + default: +#ifdef Py_DEBUG + { + printf("Unknown uop: "); + _PyUOpPrint(&next_uop[-1]); + printf(" @ %d\n", (int)(next_uop - current_executor->trace - 1)); + Py_FatalError("Unknown uop"); + } +#else + Py_UNREACHABLE(); +#endif + + } + } + +jump_to_error_target: +#ifdef Py_DEBUG + if (frame->lltrace >= 2) { + printf("Error: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(" @ %d -> %s]\n", + (int)(next_uop - current_executor->trace - 1), + _PyOpcode_OpName[frame->instr_ptr->op.code]); + } +#endif + assert(next_uop[-1].format == UOP_FORMAT_JUMP); + uint16_t target = uop_get_error_target(&next_uop[-1]); + next_uop = current_executor->trace + target; + goto tier2_dispatch; + +jump_to_jump_target: + assert(next_uop[-1].format == UOP_FORMAT_JUMP); + target = uop_get_jump_target(&next_uop[-1]); + next_uop = current_executor->trace + target; + goto tier2_dispatch; + +#endif // _Py_JIT + +#endif // _Py_TIER2 + +early_exit: + assert(_PyErr_Occurred(tstate)); + _Py_LeaveRecursiveCallPy(tstate); + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->return_offset = 0; + assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; + return NULL; +} + +#endif + +#if defined(__GNUC__) || defined(__clang__) +# pragma GCC diagnostic pop +#elif defined(_MSC_VER) /* MS_WINDOWS */ +# pragma warning(pop) +#endif + +// clang-format on +static int +_Ci_CheckArgs(PyThreadState* tstate, _PyInterpreterFrame* f, PyCodeObject* co) { + // In the future we can use co_extra to store the cached arg info + _PyStackRef* fastlocals = &f->localsplus[0]; + + PyObject* checks = _PyClassLoader_GetCodeArgumentTypeDescrs(co); + PyObject* local; + PyObject* type_descr; + PyTypeObject* type; + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(checks); i += 2) { + local = PyTuple_GET_ITEM(checks, i); + type_descr = PyTuple_GET_ITEM(checks, i + 1); + long idx = PyLong_AsLong(local); + assert(idx >= 0); + PyObject* val = PyStackRef_AsPyObjectBorrow(fastlocals[idx]); + + int optional; + int exact; + type = _PyClassLoader_ResolveType(type_descr, &optional, &exact); + if (type == NULL) { + return -1; + } + + int primitive = _PyClassLoader_GetTypeCode(type); + if (primitive == TYPED_BOOL) { + optional = 0; + Py_DECREF(type); + type = &PyBool_Type; + Py_INCREF(type); + } else if (primitive <= TYPED_INT64) { + exact = optional = 0; + Py_DECREF(type); + type = &PyLong_Type; + Py_INCREF(type); + } else if (primitive == TYPED_DOUBLE) { + exact = optional = 0; + Py_DECREF(type); + type = &PyFloat_Type; + Py_INCREF(type); + } else { + assert(primitive == TYPED_OBJECT); + } + + if (!_PyObject_TypeCheckOptional(val, type, optional, exact)) { + PyErr_Format( + CiExc_StaticTypeError, + "%U expected '%s' for argument %U, got '%s'", + co->co_name, + type->tp_name, + PyTuple_GET_ITEM(co->co_localsplusnames, idx), + Py_TYPE(val)->tp_name); + Py_DECREF(type); + return -1; + } + + Py_DECREF(type); + + if (primitive <= TYPED_INT64) { + size_t value; + if (!_PyClassLoader_OverflowCheck(val, primitive, &value)) { + PyErr_SetString(PyExc_OverflowError, "int overflow"); + return -1; + } + } + } + return 0; +} + +static PyObject* _CiStaticEval_Vector( + PyThreadState* tstate, + PyFunctionObject* func, + PyObject* locals, + PyObject* const* args, + size_t argcount, + PyObject* kwnames, + int check_args) { + size_t total_args = argcount; + if (kwnames) { + total_args += PyTuple_GET_SIZE(kwnames); + } + _PyStackRef stack_array[8]; + _PyStackRef *arguments; + if (total_args <= 8) { + arguments = stack_array; + } + else { + arguments = PyMem_Malloc(sizeof(_PyStackRef) * total_args); + if (arguments == NULL) { + return PyErr_NoMemory(); + } + } + /* _PyEvalFramePushAndInit consumes the references + * to func, locals and all its arguments */ + Py_XINCREF(locals); + for (size_t i = 0; i < argcount; i++) { + arguments[i] = PyStackRef_FromPyObjectNew(args[i]); + } + if (kwnames) { + Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); + for (Py_ssize_t i = 0; i < kwcount; i++) { + arguments[i+argcount] = PyStackRef_FromPyObjectNew(args[i+argcount]); + } + } + _PyInterpreterFrame *frame = _PyEvalFramePushAndInit( + tstate, PyStackRef_FromPyObjectNew(func), locals, + arguments, argcount, kwnames, NULL); + if (total_args > 8) { + PyMem_Free(arguments); + } + if (frame == NULL) { + return NULL; + } + + EVAL_CALL_STAT_INC(EVAL_CALL_VECTOR); +#ifdef ENABLE_INTERPRETER_LOOP + PyCodeObject* co = (PyCodeObject*)func->func_code; + assert(co->co_flags & CI_CO_STATICALLY_COMPILED); + if (check_args && _Ci_CheckArgs(tstate, frame, co) < 0) { + _PyEval_FrameClearAndPop(tstate, frame); + return NULL; + } + + return Ci_EvalFrame(tstate, frame, 0); +#else + return _PyEval_EvalFrameDefault(tstate, frame, 0); +#endif +} + +PyObject* Ci_StaticFunction_Vectorcall( + PyObject* func, + PyObject* const* stack, + size_t nargsf, + PyObject* kwnames) { + assert(PyFunction_Check(func)); + PyFunctionObject* f = (PyFunctionObject*)func; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + assert(nargs >= 0); + assert(nargs == 0 || stack != NULL); + + PyCodeObject* code = (PyCodeObject*)f->func_code; + PyObject* globals = (code->co_flags & CO_OPTIMIZED) ? NULL : f->func_globals; + + PyThreadState* tstate = _PyThreadState_GET(); + return _CiStaticEval_Vector(tstate, f, globals, stack, nargs, kwnames, 1); +} + +PyObject* _Py_HOT_FUNCTION Ci_PyFunction_CallStatic( + PyFunctionObject* func, + PyObject* const* args, + size_t nargsf, + PyObject* kwnames) { + assert(PyFunction_Check(func)); + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + assert(nargs == 0 || args != NULL); + + /* We are bound to a specific function that is known at compile time, and + * all of the arguments are guaranteed to be provided */ + PyCodeObject* co = (PyCodeObject*)func->func_code; + assert(co->co_argcount == nargs); + assert(co->co_flags & CI_CO_STATICALLY_COMPILED); + assert(co->co_flags & CO_OPTIMIZED); + assert(kwnames == NULL); + + /* Silence unused variable warnings. */ + (void)co; + (void)kwnames; + (void)nargs; + + PyThreadState* tstate = _PyThreadState_GET(); + assert(tstate != NULL); + + return _CiStaticEval_Vector(tstate, func, NULL, args, nargsf, NULL, 0); +} diff --git a/cinderx/Interpreter/gen_opcodes_314.py b/cinderx/Interpreter/gen_opcodes_314.py index 98026e90e..e6f09bbcb 100644 --- a/cinderx/Interpreter/gen_opcodes_314.py +++ b/cinderx/Interpreter/gen_opcodes_314.py @@ -23,7 +23,7 @@ template = """ // Copyright (c) Meta Platforms, Inc. and affiliates. -// 3.14 has a simple file that just defines the relavant ids: +// 3.14 has a simple file that just defines the relevant ids: #include "opcode.h" diff --git a/cinderx/Interpreter/gen_opcodes_316.py b/cinderx/Interpreter/gen_opcodes_316.py new file mode 100644 index 000000000..75c39abdd --- /dev/null +++ b/cinderx/Interpreter/gen_opcodes_316.py @@ -0,0 +1,51 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. + +import sys + +from cinderx import opcode + +STATIC_OPNAMES: list[str] = [f"<{i}>" for i in range(256)] +STATIC_OPMAP: dict[str, int] = {} +opcode.init( + STATIC_OPNAMES, + STATIC_OPMAP, + [], + [], + [], + [], + [], + {}, + {}, + {}, +) + + +template = """ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +// 3.16 has a simple file that just defines the relevant ids: + +#include "opcode.h" + +// 0x200 to make sure we don't collide with pseudo instructions +#define EXTENDED_OPCODE_FLAG 0x200 + +""" + + +def main(): + if len(sys.argv) < 2: + print("no file specified") + sys.exit(1) + + res = [template] + for name, op in STATIC_OPMAP.items(): + res.append(f"#define {name} ({op} | EXTENDED_OPCODE_FLAG)") + + with open(sys.argv[1], "w") as f: + f.write("\n".join(res)) + f.write("\n") + + +if __name__ == "__main__": + main() diff --git a/cinderx/Interpreter/interpreter.h b/cinderx/Interpreter/interpreter.h index 7402b4969..6e5b06180 100644 --- a/cinderx/Interpreter/interpreter.h +++ b/cinderx/Interpreter/interpreter.h @@ -4,7 +4,7 @@ #include "cinderx/python.h" -// Exporting Ci_PyFunction_Vectorcall. +// Exporting Ci_PyFunction_Vectorcall and getInterpretedVectorcall. #include "cinderx/module_c_state.h" #include @@ -17,15 +17,10 @@ extern "C" { /* * The CinderX frame evaluator function (interpreter loop). */ -#if PY_VERSION_HEX < 0x030C0000 -PyObject* _Py_HOT_FUNCTION -Ci_EvalFrame(PyThreadState* tstate, PyFrameObject* f, int throwflag); -#else PyObject* _Py_HOT_FUNCTION Ci_EvalFrame( PyThreadState* tstate, struct _PyInterpreterFrame* f, int throwflag); -#endif /* * General vectorcall entry point to a function compiled by the Static Python @@ -47,15 +42,6 @@ PyObject* Ci_PyFunction_CallStatic( size_t nargsf, PyObject* kwnames); -/* - * Get the appropriate entry point that will execute a function object in the - * interpreter. - * - * This is a different function for Static Python functions versus "normal" - * Python functions. - */ -vectorcallfunc getInterpretedVectorcall(const PyFunctionObject* func); - /* * Install the CinderX frame evaluator function into the runtime. */ @@ -68,9 +54,6 @@ void Ci_FiniFrameEvalFunc(); void Ci_InitOpcodes(); -extern bool Ci_DelayAdaptiveCode; -extern uint64_t Ci_AdaptiveThreshold; - #ifdef __cplusplus } #endif diff --git a/cinderx/Interpreter/interpreter_base.cpp b/cinderx/Interpreter/interpreter_base.cpp index c5281698f..8092a3913 100644 --- a/cinderx/Interpreter/interpreter_base.cpp +++ b/cinderx/Interpreter/interpreter_base.cpp @@ -1,6 +1,5 @@ // Copyright (c) Meta Platforms, Inc. and affiliates. -#include "cinderx/Common/extra-py-flags.h" #include "cinderx/Interpreter/interpreter.h" #include "cinderx/UpstreamBorrow/borrowed.h" @@ -14,26 +13,14 @@ extern "C" { -vectorcallfunc getInterpretedVectorcall( - [[maybe_unused]] const PyFunctionObject* func) { -#ifdef ENABLE_INTERPRETER_LOOP - const PyCodeObject* code = (const PyCodeObject*)(func->func_code); - return (code->co_flags & CI_CO_STATICALLY_COMPILED) - ? Ci_StaticFunction_Vectorcall - : Ci_PyFunction_Vectorcall; -#else - return Ci_PyFunction_Vectorcall; -#endif -} - int Ci_InitFrameEvalFunc() { #ifdef ENABLE_INTERPRETER_LOOP + Ci_SetStaticFunctionVectorcall(Ci_StaticFunction_Vectorcall); #ifdef ENABLE_EVAL_HOOK Ci_hook_EvalFrame = Ci_EvalFrame; #elif defined(ENABLE_PEP523_HOOK) - // Let borrowed.h know the eval frame pointer - Ci_EvalFrameFunc = Ci_EvalFrame; - + // Allow borrowed specialization code to recognize CinderX's evaluator. + Ci_SetEvalFrameFunc(Ci_EvalFrame); auto interp = _PyInterpreterState_GET(); auto current_eval_frame = _PyInterpreterState_GetEvalFrameFunc(interp); if (current_eval_frame == Ci_EvalFrame) { @@ -48,6 +35,9 @@ int Ci_InitFrameEvalFunc() { } _PyInterpreterState_SetEvalFrameFunc(interp, Ci_EvalFrame); +#if PY_VERSION_HEX >= 0x030F0000 + _PyInterpreterState_SetEvalFrameAllowSpecialization(interp, 1); +#endif #endif #endif @@ -56,10 +46,12 @@ int Ci_InitFrameEvalFunc() { void Ci_FiniFrameEvalFunc() { #ifdef ENABLE_INTERPRETER_LOOP + Ci_SetStaticFunctionVectorcall(nullptr); #ifdef ENABLE_EVAL_HOOK Ci_hook_EvalFrame = nullptr; #elif defined(ENABLE_PEP523_HOOK) _PyInterpreterState_SetEvalFrameFunc(_PyInterpreterState_GET(), nullptr); + Ci_SetEvalFrameFunc(nullptr); #endif #endif } diff --git a/cinderx/Interpreter/iter_helpers.c b/cinderx/Interpreter/iter_helpers.c index 796dc6cf5..ccef00df3 100644 --- a/cinderx/Interpreter/iter_helpers.c +++ b/cinderx/Interpreter/iter_helpers.c @@ -6,6 +6,7 @@ // clang-format off #include "internal/pycore_pyerrors.h" +#include "internal/pycore_genobject.h" PyObject* Ci_GetAIter(PyThreadState* tstate, PyObject* obj) { unaryfunc getter = NULL; @@ -76,7 +77,7 @@ PyObject* Ci_GetANext(PyThreadState* tstate, PyObject* aiter) { return NULL; } - awaitable = Cix_PyCoro_GetAwaitableIter(next_iter); + awaitable = _PyCoro_GetAwaitableIter(next_iter); if (awaitable == NULL) { _PyErr_FormatFromCause( PyExc_TypeError, diff --git a/cinderx/Interpreter/regen-cases-312.sh b/cinderx/Interpreter/regen-cases-312.sh index 35280e8b7..6847894bb 100755 --- a/cinderx/Interpreter/regen-cases-312.sh +++ b/cinderx/Interpreter/regen-cases-312.sh @@ -6,7 +6,7 @@ srcdir='../../../third-party/python/3.12' PYTHONPATH=$srcdir/Tools/cases_generator \ -buck run fbcode//cinderx:python3.10 -- \ +buck run fbcode//cinderx:python3.12 -- \ $srcdir/Tools/cases_generator/generate_cases.py \ --emit-line-directives \ -o 3.12/Includes/generated_cases.c.h \ diff --git a/cinderx/Interpreter/regen-cases-315.sh b/cinderx/Interpreter/regen-cases-315.sh index df04edf80..1bd9c2954 100755 --- a/cinderx/Interpreter/regen-cases-315.sh +++ b/cinderx/Interpreter/regen-cases-315.sh @@ -4,8 +4,8 @@ set -e # Copied from cpython Makefile -# Point srcdir to the internal copy of cpython 3.12 so we can use the cases_generator -srcdir='../../../third-party/python/main/patched' +# Point srcdir to the internal copy of cpython 3.15 so we can use the cases_generator +srcdir='../../../third-party/python/3.15/patched' PYTHONPATH=$srcdir/Tools/cases_generator \ fbpython -- \ diff --git a/cinderx/Interpreter/regen-cases-316.sh b/cinderx/Interpreter/regen-cases-316.sh new file mode 100755 index 000000000..6b94a2e32 --- /dev/null +++ b/cinderx/Interpreter/regen-cases-316.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +set -e + +# Copied from cpython Makefile +# Point srcdir to the internal copy of cpython main (3.16) so we can use the cases_generator. +# 3.16 tracks the upstream Python "main" branch, so the source lives under main/, not 3.16/. +srcdir='../../../third-party/python/main/patched' + +PYTHONPATH=$srcdir/Tools/cases_generator \ +fbpython -- \ + $srcdir/Tools/cases_generator/tier1_generator.py \ + -o 3.16/Includes/generated_cases.c.h \ + $srcdir/Python/bytecodes.c \ + 3.16/cinder-bytecodes.c + +PYTHONPATH=$srcdir/Tools/cases_generator \ +fbpython -- \ + $srcdir/Tools/cases_generator/target_generator.py \ + -o 3.16/cinderx_opcode_targets.h \ + $srcdir/Python/bytecodes.c \ + 3.16/cinder-bytecodes.c + +gen='generated' +sed -i "1i // @$gen" 3.16/Includes/generated_cases.c.h diff --git a/cinderx/Interpreter/regen-opcodes-310.sh b/cinderx/Interpreter/regen-opcodes-310.sh deleted file mode 100755 index 5902383f9..000000000 --- a/cinderx/Interpreter/regen-opcodes-310.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -# (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -OUT=$(buck build -m ovr_config//toolchain/python/constraints:3.10.cinder fbcode//cinderx/Interpreter:gen_opcode_h --show-full-simple-output) -cp "$OUT" 3.10/opcode.h - -OUT=$(buck build -m ovr_config//toolchain/python/constraints:3.10.cinder fbcode//cinderx/Interpreter:gen_cinderx_opcode_targets_h --show-full-simple-output) -cp "$OUT" 3.10/cinderx_opcode_targets.h diff --git a/cinderx/Interpreter/regen-opcodes-312.sh b/cinderx/Interpreter/regen-opcodes-312.sh index 129615c11..1e1aad62a 100755 --- a/cinderx/Interpreter/regen-opcodes-312.sh +++ b/cinderx/Interpreter/regen-opcodes-312.sh @@ -1,11 +1,11 @@ #!/usr/bin/env bash # (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. -OUT=$(buck build -m ovr_config//toolchain/python/constraints:3.12 fbcode//cinderx/Interpreter:gen_opcode_h --show-full-simple-output) +OUT=$(buck build -m 'ovr_config//toolchain/python/constraints:python-version[3.12]' fbcode//cinderx/Interpreter:gen_opcode_h --show-full-simple-output) cp "$OUT" 3.12/opcode.h -OUT=$(buck build -m ovr_config//toolchain/python/constraints:3.12 fbcode//cinderx/Interpreter:gen_opcode_h\[cinder_opcode.h\] --show-full-simple-output) +OUT=$(buck build -m 'ovr_config//toolchain/python/constraints:python-version[3.12]' fbcode//cinderx/Interpreter:gen_opcode_h\[cinder_opcode.h\] --show-full-simple-output) cp "$OUT" 3.12/cinder_opcode_metadata.h -OUT=$(buck build -m ovr_config//toolchain/python/constraints:3.12 fbcode//cinderx/Interpreter:gen_cinderx_opcode_targets_h --show-full-simple-output) +OUT=$(buck build -m 'ovr_config//toolchain/python/constraints:python-version[3.12]' fbcode//cinderx/Interpreter:gen_cinderx_opcode_targets_h --show-full-simple-output) cp "$OUT" 3.12/cinderx_opcode_targets.h diff --git a/cinderx/Interpreter/regen-opcodes-314.sh b/cinderx/Interpreter/regen-opcodes-314.sh index 6b6c8bc2b..f7bb463ce 100755 --- a/cinderx/Interpreter/regen-opcodes-314.sh +++ b/cinderx/Interpreter/regen-opcodes-314.sh @@ -4,6 +4,6 @@ set -e root=$(sl root) -buck2 run fbcode//cinderx/PythonLib/opcodes:assign_opcode_numbers314 $root/fbcode/cinderx/PythonLib/opcodes/3.14/opcode.py +buck2 run fbcode//cinderx/PythonLib/opcodes:assign_opcode_numbers314 -- $root/fbcode/cinderx/PythonLib/opcodes/3_14/opcode.py -buck2 run :gen-opcodes-314 $root/fbcode/cinderx/Interpreter/3.14/cinder_opcode_ids.h +buck2 run :gen-opcodes-314 -- $root/fbcode/cinderx/Interpreter/3.14/cinder_opcode_ids.h diff --git a/cinderx/Interpreter/regen-opcodes-315.sh b/cinderx/Interpreter/regen-opcodes-315.sh index 6b6c8bc2b..f7bb463ce 100755 --- a/cinderx/Interpreter/regen-opcodes-315.sh +++ b/cinderx/Interpreter/regen-opcodes-315.sh @@ -4,6 +4,6 @@ set -e root=$(sl root) -buck2 run fbcode//cinderx/PythonLib/opcodes:assign_opcode_numbers314 $root/fbcode/cinderx/PythonLib/opcodes/3.14/opcode.py +buck2 run fbcode//cinderx/PythonLib/opcodes:assign_opcode_numbers314 -- $root/fbcode/cinderx/PythonLib/opcodes/3_14/opcode.py -buck2 run :gen-opcodes-314 $root/fbcode/cinderx/Interpreter/3.14/cinder_opcode_ids.h +buck2 run :gen-opcodes-314 -- $root/fbcode/cinderx/Interpreter/3.14/cinder_opcode_ids.h diff --git a/cinderx/Interpreter/regen-opcodes-316.sh b/cinderx/Interpreter/regen-opcodes-316.sh new file mode 100755 index 000000000..2157d75ab --- /dev/null +++ b/cinderx/Interpreter/regen-opcodes-316.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +set -e + +root=$(sl root) +buck2 run fbcode//cinderx/PythonLib/opcodes:assign_opcode_numbers316 -- "$root/fbcode/cinderx/PythonLib/opcodes/3_16/opcode.py" + +buck2 run :gen-opcodes-316 -- "$root/fbcode/cinderx/Interpreter/3.16/cinder_opcode_ids.h" diff --git a/cinderx/Jit/anextawaitable.cpp b/cinderx/Jit/anextawaitable.cpp new file mode 100644 index 000000000..eff4a4448 --- /dev/null +++ b/cinderx/Jit/anextawaitable.cpp @@ -0,0 +1,187 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#include "cinderx/Jit/anextawaitable.h" + +#include "internal/pycore_genobject.h" +#include "internal/pycore_object.h" + +#include "cinderx/Common/log.h" +#include "cinderx/Common/ref.h" +#include "cinderx/Jit/generators_rt.h" +#include "cinderx/module_state.h" + +namespace cinderx::jit { + +namespace { + +struct ANextAwaitableObject { + PyObject_HEAD + PyObject* wrapped; + PyObject* default_value; +}; + +Ref<> anextawaitable_getiter(ANextAwaitableObject* obj) { + JIT_DCHECK(obj->wrapped != nullptr, "anextawaitable has no wrapped object"); + Ref<> awaitable = Ref<>::steal(JitCoro_GetAwaitableIter(obj->wrapped)); + if (awaitable == nullptr) { + return nullptr; + } + if (Py_TYPE(awaitable)->tp_iternext == nullptr) { + // JitCoro_GetAwaitableIter returns a Coroutine, a Generator, + // or an iterator. Of these, only coroutines lack tp_iternext. + JIT_DCHECK( + JitCoro_CheckExact(awaitable) || PyCoro_CheckExact(awaitable), + "awaitable without tp_iternext should be a coroutine"); + unaryfunc getter = Py_TYPE(awaitable)->tp_as_async->am_await; + Ref<> new_awaitable = Ref<>::steal(getter(awaitable)); + if (new_awaitable == nullptr) { + return nullptr; + } + awaitable = std::move(new_awaitable); + if (!PyIter_Check(awaitable)) { + PyErr_Format( + PyExc_TypeError, + "%T.__await__() must return an iterable, not %T", + obj, + awaitable.get()); + return nullptr; + } + } + return awaitable; +} + +PyObject* anextawaitable_proxy( + ANextAwaitableObject* obj, + const char* meth, + PyObject* arg) { + Ref<> awaitable = anextawaitable_getiter(obj); + if (awaitable == nullptr) { + return nullptr; + } + // When specified, 'arg' may be a tuple (if coming from a METH_VARARGS + // method) or a single object (if coming from a METH_O method). + Ref<> ret = Ref<>::steal( + arg == nullptr ? PyObject_CallMethod(awaitable, meth, nullptr) + : PyObject_CallMethod(awaitable, meth, "O", arg)); + if (ret != nullptr) { + return ret.release(); + } + if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration)) { + // ANextAwaitableObject is only used by anext() when a default value is + // provided. So when we have a StopAsyncIteration exception we replace it + // with a StopIteration(default), as if it was the return value of + // __anext__() coroutine. + PyErr_Clear(); + _PyGen_SetStopIterationValue(obj->default_value); + } + return nullptr; +} + +void anextawaitable_dealloc(ANextAwaitableObject* obj) { + PyTypeObject* type = Py_TYPE(obj); + PyObject_GC_UnTrack(obj); + Py_XDECREF(obj->wrapped); + Py_XDECREF(obj->default_value); + PyObject_GC_Del(obj); + // Heap types increment their type, so we need to decrement it here: + Py_DECREF(type); +} + +int anextawaitable_traverse( + ANextAwaitableObject* obj, + visitproc visit, + void* arg) { + Py_VISIT(obj->wrapped); + Py_VISIT(obj->default_value); + return 0; +} + +PyObject* anextawaitable_iternext(ANextAwaitableObject* obj) { + Ref<> awaitable = anextawaitable_getiter(obj); + if (awaitable == nullptr) { + return nullptr; + } + Ref<> result = Ref<>::steal((*Py_TYPE(awaitable)->tp_iternext)(awaitable)); + if (result != nullptr) { + return result.release(); + } + if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration)) { + PyErr_Clear(); + _PyGen_SetStopIterationValue(obj->default_value); + } + return nullptr; +} + +PyObject* anextawaitable_send(ANextAwaitableObject* obj, PyObject* arg) { + return anextawaitable_proxy(obj, "send", arg); +} + +PyObject* anextawaitable_throw(ANextAwaitableObject* obj, PyObject* arg) { + return anextawaitable_proxy(obj, "throw", arg); +} + +PyObject* anextawaitable_close( + ANextAwaitableObject* obj, + [[maybe_unused]] PyObject* arg) { + return anextawaitable_proxy(obj, "close", nullptr); +} + +PyMethodDef anextawaitable_methods[] = { + {"send", reinterpret_cast(anextawaitable_send), METH_O, ""}, + {"throw", + reinterpret_cast(anextawaitable_throw), + METH_VARARGS, + ""}, +#if PY_VERSION_HEX >= 0x030E0000 + {"close", + reinterpret_cast(anextawaitable_close), + METH_NOARGS, + ""}, +#else + {"close", + reinterpret_cast(anextawaitable_close), + METH_VARARGS, + ""}, +#endif + {} // Sentinel +}; + +} // namespace + +PyType_Slot anext_awaitable_slots[] = { + {Py_tp_dealloc, reinterpret_cast(anextawaitable_dealloc)}, + {Py_tp_traverse, reinterpret_cast(anextawaitable_traverse)}, + {Py_tp_methods, anextawaitable_methods}, + {Py_tp_iternext, reinterpret_cast(anextawaitable_iternext)}, + {Py_tp_iter, reinterpret_cast(PyObject_SelfIter)}, + {Py_am_await, reinterpret_cast(PyObject_SelfIter)}, + {0, nullptr}, +}; + +PyType_Spec JitAnextAwaitable_Spec = { + .name = "builtins.anext_awaitable", + .basicsize = sizeof(ANextAwaitableObject), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE, + .slots = anext_awaitable_slots, +}; + +// This is the same as CPython's anextawaitable except it also recognizes +// JIT coroutines. Most of the code is CPython's implementation but converted +// to C++. This code is small, rarely changes, and we use it across multiple +// versions so it's just easier to maintain our own version. +PyObject* JitGen_AnextAwaitable_New( + cinderx::ModuleState* moduleState, + PyObject* awaitable, + PyObject* defaultValue) { + ANextAwaitableObject* anext = + PyObject_GC_New(ANextAwaitableObject, moduleState->anext_awaitable_type); + if (anext == nullptr) { + return nullptr; + } + anext->wrapped = Py_NewRef(awaitable); + anext->default_value = Py_NewRef(defaultValue); + PyObject_GC_Track(anext); + return reinterpret_cast(anext); +} + +} // namespace cinderx::jit diff --git a/cinderx/Jit/anextawaitable.h b/cinderx/Jit/anextawaitable.h new file mode 100644 index 000000000..18c4a3336 --- /dev/null +++ b/cinderx/Jit/anextawaitable.h @@ -0,0 +1,18 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#pragma once + +#include "cinderx/python.h" + +#include "cinderx/module_state.h" + +namespace cinderx::jit { + +extern PyType_Spec JitAnextAwaitable_Spec; + +PyObject* JitGen_AnextAwaitable_New( + cinderx::ModuleState* moduleState, + PyObject* awaitable, + PyObject* defaultValue); + +} // namespace cinderx::jit diff --git a/cinderx/Jit/bitvector.cpp b/cinderx/Jit/bitvector.cpp index 57e495e82..33b822bf5 100644 --- a/cinderx/Jit/bitvector.cpp +++ b/cinderx/Jit/bitvector.cpp @@ -2,339 +2,370 @@ #include "cinderx/Jit/bitvector.h" +#include "cinderx/Common/log.h" + #include +#include #include -namespace jit::util { +namespace cinderx::jit::util { + +namespace { + +constexpr size_t kChunkBitWidth = sizeof(uint64_t) * CHAR_BIT; + +// Get the number of chunks needed to fit a specific bit width. +constexpr size_t chunksForBits(size_t num_bits) { + return num_bits / kChunkBitWidth + (num_bits % kChunkBitWidth == 0 ? 0 : 1); +} + +} // namespace BitVector::~BitVector() { - if (!IsShortVector()) { - delete bits_.bit_vec; + if (!isShortVector()) { + delete bit_vec; } } -BitVector::BitVector(size_t nb) : num_bits_(nb) { - if (IsShortVector()) { - bits_.bits = 0; +BitVector::BitVector(size_t num_bits) : num_bits_{num_bits} { + if (isShortVector()) { + bits = 0; } else { - size_t size = num_bits_ / PTR_WIDTH + (num_bits_ % PTR_WIDTH == 0 ? 0 : 1); - bits_.bit_vec = new std::vector(size, 0); + bit_vec = new std::vector(chunksForBits(num_bits), 0); } } -BitVector& BitVector::operator=(const BitVector& bv) { - if (this == &bv) { +BitVector::BitVector(const BitVector& bv) { + *this = bv; +} + +BitVector::BitVector(BitVector&& bv) noexcept { + *this = std::move(bv); +} + +BitVector& BitVector::operator=(const BitVector& rhs) { + if (this == &rhs) { return *this; } - bool lhs_short = IsShortVector(); - bool rhs_short = bv.IsShortVector(); + bool lhs_short = isShortVector(); + bool rhs_short = rhs.isShortVector(); - num_bits_ = bv.num_bits_; + num_bits_ = rhs.num_bits_; if (lhs_short && rhs_short) { - bits_.bits = bv.bits_.bits; + bits = rhs.bits; } else if (lhs_short && !rhs_short) { - bits_.bit_vec = new std::vector(*bv.bits_.bit_vec); + bit_vec = new std::vector(*rhs.bit_vec); } else if (!lhs_short && rhs_short) { - delete bits_.bit_vec; - bits_.bits = bv.bits_.bits; + delete bit_vec; + bits = rhs.bits; } else { // if (!lhs_short && !rhs_short) - *bits_.bit_vec = *bv.bits_.bit_vec; + *bit_vec = *rhs.bit_vec; } return *this; } -BitVector& BitVector::operator=(BitVector&& bv) { - if (this == &bv) { +BitVector& BitVector::operator=(BitVector&& rhs) noexcept { + if (this == &rhs) { return *this; } - if (!IsShortVector()) { - delete bits_.bit_vec; + if (!isShortVector()) { + delete bit_vec; } - num_bits_ = bv.num_bits_; - bits_ = bv.bits_; - bv.num_bits_ = 0; + num_bits_ = rhs.num_bits_; + if (rhs.isShortVector()) { + bits = rhs.bits; + } else { + bit_vec = rhs.bit_vec; + } + rhs.num_bits_ = 0; + rhs.bits = 0; return *this; } bool BitVector::operator==(const BitVector& rhs) const { - JIT_CHECK(num_bits_ == rhs.num_bits_, "LHS and RHS are of different widths."); - - if (IsShortVector()) { - return bits_.bits == rhs.bits_.bits; - } - - return std::equal( - bits_.bit_vec->begin(), bits_.bit_vec->end(), rhs.bits_.bit_vec->begin()); + JIT_THROW_IF( + num_bits_ != rhs.num_bits_, + "Comparing bitvectors of different widths, {} and {}", + num_bits_, + rhs.num_bits_); + auto cs = chunks(); + return std::equal(cs.begin(), cs.end(), rhs.chunks().begin()); } template -BitVector BitVector::BinaryOp(const BitVector& rhs, const Op& op) const { - JIT_CHECK(num_bits_ == rhs.num_bits_, "LHS and RHS are of different widths."); - - if (IsShortVector()) { - return BitVector(num_bits_, op(bits_.bits, rhs.bits_.bits)); +BitVector BitVector::binaryOp(const BitVector& rhs, const Op& op) const { + JIT_THROW_IF( + num_bits_ != rhs.num_bits_, + "Binary operation on bitvectors of different widths, {} and {}", + num_bits_, + rhs.num_bits_); + + if (isShortVector()) { + return BitVector{num_bits_, op(bits, rhs.bits)}; } BitVector bv; bv.num_bits_ = num_bits_; - bv.bits_.bit_vec = new std::vector; - bv.bits_.bit_vec->reserve(bits_.bit_vec->size()); + bv.bit_vec = new std::vector{}; + bv.bit_vec->reserve(bit_vec->size()); std::transform( - bits_.bit_vec->begin(), - bits_.bit_vec->end(), - rhs.bits_.bit_vec->begin(), - std::back_inserter(*bv.bits_.bit_vec), - [op](uint64_t a, uint64_t b) -> uint64_t { return op(a, b); }); + bit_vec->begin(), + bit_vec->end(), + rhs.bit_vec->begin(), + std::back_inserter(*bv.bit_vec), + op); return bv; } BitVector BitVector::operator&(const BitVector& rhs) const { - return BinaryOp( + return binaryOp( rhs, [](uint64_t a, uint64_t b) -> uint64_t { return a & b; }); } BitVector BitVector::operator|(const BitVector& rhs) const { - return BinaryOp( + return binaryOp( rhs, [](uint64_t a, uint64_t b) -> uint64_t { return a | b; }); } BitVector BitVector::operator-(const BitVector& rhs) const { - return BinaryOp( + return binaryOp( rhs, [](uint64_t a, uint64_t b) -> uint64_t { return a & ~b; }); } template -BitVector& BitVector::BinaryOpAssign(const BitVector& rhs, const Op& op) { - JIT_CHECK(num_bits_ == rhs.num_bits_, "LHS and RHS are of different widths."); - - if (IsShortVector()) { - bits_.bits = op(bits_.bits, rhs.bits_.bits); +BitVector& BitVector::binaryOpAssign(const BitVector& rhs, const Op& op) { + JIT_THROW_IF( + num_bits_ != rhs.num_bits_, + "Binary operation on bitvectors of different widths, {} and {}", + num_bits_, + rhs.num_bits_); + + if (isShortVector()) { + bits = op(bits, rhs.bits); } else { std::transform( - bits_.bit_vec->begin(), - bits_.bit_vec->end(), - rhs.bits_.bit_vec->begin(), - bits_.bit_vec->begin(), - [op](uint64_t a, uint64_t b) -> uint64_t { return op(a, b); }); + bit_vec->begin(), + bit_vec->end(), + rhs.bit_vec->begin(), + bit_vec->begin(), + op); } return *this; } BitVector& BitVector::operator&=(const BitVector& rhs) { - return BinaryOpAssign( + return binaryOpAssign( rhs, [](uint64_t a, uint64_t b) -> uint64_t { return a & b; }); } BitVector& BitVector::operator|=(const BitVector& rhs) { - return BinaryOpAssign( + return binaryOpAssign( rhs, [](uint64_t a, uint64_t b) -> uint64_t { return a | b; }); } BitVector& BitVector::operator-=(const BitVector& rhs) { - return BinaryOpAssign( + return binaryOpAssign( rhs, [](uint64_t a, uint64_t b) -> uint64_t { return a & ~b; }); } -void BitVector::ResetAll() { - if (IsShortVector()) { - bits_.bits = 0; - } else { - for (auto& v : *(bits_.bit_vec)) { - v = 0; - } +void BitVector::resetAll() { + for (uint64_t& chunk : chunks()) { + chunk = 0; } } -void BitVector::fill(bool v) { - if (!v) { - return ResetAll(); +void BitVector::setAll() { + auto cs = chunks(); + for (uint64_t& chunk : cs) { + chunk = ~uint64_t{0}; } - if (IsShortVector()) { - if (num_bits_ == PTR_WIDTH) { - bits_.bits = -1; - } else { - bits_.bits = (uintptr_t{1} << num_bits_) - 1; - } - } else { - auto& vec = *bits_.bit_vec; - for (size_t i = 0; i < vec.size() - 1; ++i) { - vec[i] = -1; - } - - auto remainder = num_bits_ % PTR_WIDTH; - if (remainder == 0) { - vec.back() = -1; - } else { - vec.back() = (uintptr_t{1} << remainder) - 1; - } + auto remainder = num_bits_ % kChunkBitWidth; + if (cs.size() > 0 && remainder != 0) { + cs.back() = (uint64_t{1} << remainder) - 1; } } -void BitVector::SetBit(size_t bit, bool v) { - JIT_CHECK(bit < num_bits_, "bit is too large."); - if (IsShortVector()) { - auto b = uintptr_t(1) << bit; - bits_.bits = v ? (bits_.bits | b) : (bits_.bits & ~b); +void BitVector::fill(bool v) { + if (v) { + setAll(); } else { - size_t index = bit / PTR_WIDTH; - size_t offset = bit % PTR_WIDTH; - auto& val = bits_.bit_vec->at(index); - uintptr_t b = uintptr_t(1) << offset; - val = v ? (val | b) : (val & ~b); + resetAll(); } } -size_t BitVector::AddBits(size_t i) { - auto new_num_bits = num_bits_ + i; - SetBitWidth(new_num_bits); - return new_num_bits; +bool BitVector::getBit(size_t bit) const { + JIT_THROW_IF( + bit >= num_bits_, + "BitVector::getBit() called on bit {} for vector of size {}", + bit, + num_bits_); + size_t chunk = bit / kChunkBitWidth; + size_t offset = bit % kChunkBitWidth; + return chunks()[chunk] & (uint64_t{1} << offset); } -void BitVector::SetBitWidth(size_t size) { - if (num_bits_ == size) { - return; - } - - bool old_short = IsShortVector(); - auto new_num_bits = size; - num_bits_ = new_num_bits; - bool new_short = IsShortVector(); - - if (old_short && !new_short) { - size_t size_2 = - num_bits_ / PTR_WIDTH + (num_bits_ % PTR_WIDTH == 0 ? 0 : 1); - - auto old_bits = bits_.bits; - bits_.bit_vec = new std::vector(size_2); - bits_.bit_vec->at(0) = old_bits; - } else if (!old_short && !new_short) { - size_t size_2 = - num_bits_ / PTR_WIDTH + (num_bits_ % PTR_WIDTH == 0 ? 0 : 1); - bits_.bit_vec->resize(size_2); - } else if (!old_short && new_short) { - auto low_bits = bits_.bit_vec->at(0); - delete bits_.bit_vec; - bits_.bits = low_bits; - } +void BitVector::setBit(size_t bit, bool v) { + JIT_THROW_IF( + bit >= num_bits_, + "BitVector::setBit() called on bit {} for vector of size {}", + bit, + num_bits_); - // need to clear the unused upper bits - // could use BZHI instruction, but this function is not frequently called, - // so it is okay. - auto high_mask = (uint64_t(1) << (num_bits_ % PTR_WIDTH)) - 1; - if (new_short) { - bits_.bits &= high_mask; + if (isShortVector()) { + auto b = uint64_t{1} << bit; + bits = v ? (bits | b) : (bits & ~b); } else { - auto& chunk = *bits_.bit_vec->rbegin(); - chunk &= high_mask; + size_t index = bit / kChunkBitWidth; + size_t offset = bit % kChunkBitWidth; + auto& val = bit_vec->at(index); + auto b = uint64_t{1} << offset; + val = v ? (val | b) : (val & ~b); } } -bool BitVector::GetBit(size_t bit) const { - JIT_CHECK(bit < num_bits_, "bit is out of range."); - if (IsShortVector()) { - auto b = uintptr_t(1) << bit; - return bits_.bits & b; - } - - size_t index = bit / PTR_WIDTH; - size_t offset = bit % PTR_WIDTH; - - return bits_.bit_vec->at(index) & (uintptr_t(1) << offset); +uint64_t BitVector::shortBits() const { + JIT_THROW_IF( + !isShortVector(), "BitVector::shortBits() called on large vector"); + return bits; } -void BitVector::forEachSetBit(std::function per_bit_func) const { - auto forEachBitInChunk = [&](uint64_t chunk, size_t base) { - while (chunk) { - int bit = __builtin_ctzl(chunk); - chunk ^= chunk & -chunk; - per_bit_func(bit + base); - } - }; - - if (IsShortVector()) { - forEachBitInChunk(bits_.bits, 0); - } else { - size_t chunk_base = 0; - for (uint64_t chunk : *bits_.bit_vec) { - forEachBitInChunk(chunk, chunk_base); - chunk_base += PTR_WIDTH; - } - } +void BitVector::setShortBits(uint64_t new_bits) { + JIT_THROW_IF( + !isShortVector(), + "BitVector::setShortBits() with value {} on large vector of size {}", + new_bits, + num_bits_); + JIT_THROW_IF( + num_bits_ != kChunkBitWidth && + (new_bits & ~((uint64_t{1} << num_bits_) - 1)) != 0, + "BitVector::setShortBits() with value {} can't fit in vector of size {}", + new_bits, + num_bits_); + bits = new_bits; } -uint64_t BitVector::GetBitChunk(size_t chunk) const { - if (IsShortVector()) { - JIT_CHECK(chunk == 0, "chunk is out of range."); - return bits_.bits; - } - - JIT_CHECK(chunk < bits_.bit_vec->size(), "chunk is out of range."); - return bits_.bit_vec->at(chunk); +uint64_t BitVector::getBitChunk(size_t chunk) const { + auto cs = chunks(); + JIT_THROW_IF( + chunk >= cs.size(), + "BitVector::getBitChunk() with chunk {} but vector has {} chunks", + chunk, + cs.size()); + return cs[chunk]; } -void BitVector::SetBitChunk(size_t chunk, uint64_t bits) { - auto num_chunks = (num_bits_ + PTR_WIDTH - 1) / PTR_WIDTH; - JIT_CHECK(chunk < num_chunks, "chunk is out of range"); +void BitVector::setBitChunk(size_t chunk, uint64_t bits) { + auto cs = chunks(); + auto num_chunks = cs.size(); + JIT_THROW_IF( + chunk >= cs.size(), + "BitVector::setBitChunk() with chunk {} but vector has {} chunks", + chunk, + num_chunks); if (chunk == num_chunks - 1) { - auto remainder = num_bits_ % PTR_WIDTH; + auto remainder = num_bits_ % kChunkBitWidth; if (remainder != 0) { auto mask = ~((uint64_t{1} << remainder) - 1); - JIT_CHECK((mask & bits) == 0, "invalid bit chunk"); + JIT_THROW_IF( + (mask & bits) != 0, + "BitVector::setBitChunk() on final chunk {} but value {} is too big " + "for vector bitsize {}", + chunk, + bits, + num_bits_); } } - if (IsShortVector()) { - bits_.bits = bits; + cs[chunk] = bits; +} + +void BitVector::setBitWidth(size_t size) { + if (num_bits_ == size) { return; } - (*bits_.bit_vec)[chunk] = bits; -} + bool old_short = isShortVector(); + auto new_num_bits = size; + num_bits_ = new_num_bits; + bool new_short = isShortVector(); + + if (old_short && !new_short) { + size_t size_2 = chunksForBits(num_bits_); + auto old_bits = bits; + bit_vec = new std::vector(size_2); + bit_vec->at(0) = old_bits; + } else if (!old_short && !new_short) { + size_t size_2 = chunksForBits(num_bits_); + bit_vec->resize(size_2); + } else if (!old_short && new_short) { + auto low_bits = bit_vec->at(0); + delete bit_vec; + bits = low_bits; + } -size_t BitVector::GetPopCount() const { - if (IsShortVector()) { - return __builtin_popcountll(bits_.bits); + // Clear the unused upper bits of the last chunk. Could use the BZHI + // instruction, but this function is not frequently called, so it is okay. + if (auto remainder = num_bits_ % kChunkBitWidth; remainder != 0) { + auto high_mask = (uint64_t{1} << remainder) - 1; + if (new_short) { + bits &= high_mask; + } else { + auto& chunk = bit_vec->back(); + chunk &= high_mask; + } } +} +size_t BitVector::getNumBits() const { + return num_bits_; +} + +size_t BitVector::getPopCount() const { size_t count = 0; - for (auto& b : *bits_.bit_vec) { - count += __builtin_popcountll(b); + for (uint64_t chunk : chunks()) { + count += std::popcount(chunk); } return count; } -bool BitVector::IsEmpty() const { - if (IsShortVector()) { - return bits_.bits == 0; - } +bool BitVector::isEmpty() const { + return std::ranges::all_of( + chunks(), [](uint64_t chunk) { return chunk == 0; }); +} - for (auto& b : *bits_.bit_vec) { - if (b != 0) { - return false; - } - } - return true; +std::span BitVector::chunks() { + return isShortVector() ? std::span{&bits, chunksForBits(num_bits_)} + : std::span{bit_vec->data(), bit_vec->size()}; +} + +std::span BitVector::chunks() const { + auto mutable_span = const_cast(this)->chunks(); + return {mutable_span.begin(), mutable_span.size()}; +} + +bool BitVector::isShortVector() const { + return num_bits_ <= kChunkBitWidth; } std::ostream& operator<<(std::ostream& os, const BitVector& bv) { os << '['; - for (std::size_t i = 0, n = bv.GetNumBits(); i < n; ++i) { + for (std::size_t i = 0, n = bv.getNumBits(); i < n; ++i) { if (i > 0 && (i % 8) == 0) { os << ';'; } - os << (bv.GetBit(i) ? '1' : '0'); + os << (bv.getBit(i) ? '1' : '0'); } os << ']'; return os; } -} // namespace jit::util +} // namespace cinderx::jit::util diff --git a/cinderx/Jit/bitvector.h b/cinderx/Jit/bitvector.h index 59c7c16ce..60d68c116 100644 --- a/cinderx/Jit/bitvector.h +++ b/cinderx/Jit/bitvector.h @@ -2,57 +2,39 @@ #pragma once -#include "cinderx/Common/log.h" -#include "fmt/ostream.h" +#include +#include #include #include -#include #include -#include -#include +#include #include -namespace jit::util { +namespace cinderx::jit::util { class BitVector { public: - BitVector() : num_bits_(0) { - bits_.bits = 0; - } - + BitVector() = default; ~BitVector(); - template - BitVector(size_t nb, T val) { - static_assert(std::is_integral_v, "val must be of an integral type."); - JIT_CHECK(nb <= sizeof(void*) * 8, "Bit width is too large.") - JIT_CHECK( - nb == 64 || (val & ~((T{1} << nb) - 1)) == 0, - "Val has too many bits for bit width"); - num_bits_ = nb; - bits_.bits = val; - } - - /* implicit */ BitVector(size_t size); + /* implicit */ BitVector(size_t num_bits); - BitVector(const BitVector& bv) : num_bits_(0) { - *this = bv; - } - BitVector(BitVector&& bv) : num_bits_(0) { - *this = std::move(bv); + template + BitVector(size_t num_bits, T bits) : BitVector{num_bits} { + setShortBits(bits); } - BitVector& operator=(const BitVector& bv); - BitVector& operator=(BitVector&& bv); + BitVector(const BitVector& bv); + BitVector(BitVector&& bv) noexcept; + + BitVector& operator=(const BitVector& rhs); + BitVector& operator=(BitVector&& rhs) noexcept; // Operators for the bit vector. Due to the purpose of this class (used in DFG // analysis), we only support operations between two bit vectors with the same // width. bool operator==(const BitVector& rhs) const; - bool operator!=(const BitVector& rhs) const { - return !(*this == rhs); - } BitVector operator&(const BitVector& rhs) const; BitVector operator|(const BitVector& rhs) const; BitVector operator-(const BitVector& rhs) const; @@ -60,64 +42,80 @@ class BitVector { BitVector& operator|=(const BitVector& rhs); BitVector& operator-=(const BitVector& rhs); - // Reset all bits to 0 with num_bits_ unchanged. - void ResetAll(); + // Reset all bits to 0. + void resetAll(); + // Set all bits to 1. + void setAll(); - // Set all bits to v. + // Set all bits to `v`. void fill(bool v); - // Get and set a bit in the position specified in bit. The bit index should - // be in the range of the bit vector, i.e. less than num_bits_. - bool GetBit(size_t bit) const; + // Get and set a bit in the position specified in bit. The bit index must + // be in the range of the bit vector. + bool getBit(size_t bit) const; + void setBit(size_t bit, bool v = true); + + // Run a function for every set bit, passing it the bit index. + template + requires std::invocable + void forEachSetBit(F&& per_bit_func) const { + size_t chunk_base = 0; + for (uint64_t chunk : chunks()) { + while (chunk > 0) { + int bit = std::countr_zero(chunk); + chunk ^= chunk & -chunk; + per_bit_func(bit + chunk_base); + } + chunk_base += sizeof(uint64_t) * CHAR_BIT; + } + } - void forEachSetBit(std::function per_bit_func) const; + // Get and set the bit vector as a uint64_t. Only works if its size is small + // enough to fit in a uint64_t. + uint64_t shortBits() const; + void setShortBits(uint64_t bits); - void SetBit(size_t bit, bool v = true); // Get or set a 64-bit chunk of bits. - uint64_t GetBitChunk(size_t chunk = 0) const; - void SetBitChunk(size_t chunk, uint64_t bits); + uint64_t getBitChunk(size_t chunk = 0) const; + void setBitChunk(size_t chunk, uint64_t bits); - // Add number of bits specified in i to the bit vector. Returns the new size. - size_t AddBits(size_t i); // Resize the bit vector to the number of bits specified in size. If size is // less than the current number of bits, the bit vector will be truncated. - void SetBitWidth(size_t size); + void setBitWidth(size_t size); - size_t GetNumBits() const { - return num_bits_; - } - size_t GetPopCount() const; - bool IsEmpty() const; + size_t getNumBits() const; + size_t getPopCount() const; + bool isEmpty() const; private: - size_t num_bits_; + size_t num_bits_{0}; /* * For a bit vector <= 64 bits (which is the bit width of a pointer), the bits - * are saved in bits. For a larger bit vector, it is divided into 64-bit - * chunks and saved in bit_vec. + * are saved inline. For a larger bit vector, it is divided into 64-bit + * chunks and saved in a vector. */ union { - uintptr_t bits; + uint64_t bits{0}; std::vector* bit_vec; - } bits_; + }; - static constexpr size_t PTR_WIDTH = sizeof(void*) * 8; + std::span chunks(); + std::span chunks() const; - bool IsShortVector() const { - return num_bits_ <= PTR_WIDTH; - } + bool isShortVector() const; template - BitVector BinaryOp(const BitVector& rhs, const Op& op) const; + BitVector binaryOp(const BitVector& rhs, const Op& op) const; template - BitVector& BinaryOpAssign(const BitVector& rhs, const Op& op); + BitVector& binaryOpAssign(const BitVector& rhs, const Op& op); }; std::ostream& operator<<(std::ostream& os, const BitVector& bv); -} // namespace jit::util +} // namespace cinderx::jit::util template <> -struct fmt::formatter : fmt::ostream_formatter {}; +struct fmt::formatter : fmt::ostream_formatter { +}; diff --git a/cinderx/Jit/bytecode.cpp b/cinderx/Jit/bytecode.cpp index 8bc356759..8e43ad8dd 100644 --- a/cinderx/Jit/bytecode.cpp +++ b/cinderx/Jit/bytecode.cpp @@ -2,7 +2,7 @@ #include "cinderx/Jit/bytecode.h" -namespace jit { +namespace cinderx::jit { BCOffset BytecodeInstruction::baseOffset() const { return baseOffset_; @@ -71,7 +71,6 @@ int BytecodeInstruction::uninstrumentedOpcode() const { } int BytecodeInstruction::specializedOpcode() const { -#if PY_VERSION_HEX >= 0x030C0000 int opcode = uninstrumentedOpcode(); switch (opcode) { @@ -80,6 +79,9 @@ int BytecodeInstruction::specializedOpcode() const { case BINARY_OP_ADD_UNICODE: case BINARY_OP_MULTIPLY_FLOAT: case BINARY_OP_MULTIPLY_INT: + case BINARY_OP_SUBSCR_DICT: + case BINARY_OP_SUBSCR_LIST_INT: + case BINARY_OP_SUBSCR_TUPLE_INT: case BINARY_OP_SUBTRACT_FLOAT: case BINARY_OP_SUBTRACT_INT: case BINARY_SUBSCR_DICT: @@ -90,6 +92,12 @@ int BytecodeInstruction::specializedOpcode() const { case COMPARE_OP_STR: case LOAD_ATTR_MODULE: case STORE_SUBSCR_DICT: + case STORE_SUBSCR_LIST_INT: + case TO_BOOL_BOOL: + case TO_BOOL_INT: + case TO_BOOL_LIST: + case TO_BOOL_NONE: + case TO_BOOL_STR: case UNPACK_SEQUENCE_LIST: case UNPACK_SEQUENCE_TUPLE: case UNPACK_SEQUENCE_TWO_TUPLE: @@ -97,9 +105,6 @@ int BytecodeInstruction::specializedOpcode() const { default: return unspecialize(opcode); } -#else - return opcode(); -#endif } int BytecodeInstruction::oparg() const { @@ -133,6 +138,10 @@ bool BytecodeInstruction::isBranch() const { } } +bool BytecodeInstruction::isBackwardBranch() const { + return isBranch() && getJumpTarget() <= baseOffset(); +} + bool BytecodeInstruction::isReturn() const { switch (opcode()) { case RETURN_CONST: @@ -176,7 +185,7 @@ BCOffset BytecodeInstruction::getJumpTarget() const { // We make this tweak here so it applies both when generating the branching // HIR operation, and when creating block boundaries for bytecode. The END_FOR // will end up on its own in an unreachable block. - if (PY_VERSION_HEX >= 0x030B0000 && opcode() == FOR_ITER) { + if (opcode() == FOR_ITER) { BytecodeInstruction target_bc{code_, target}; JIT_CHECK(target_bc.opcode() == END_FOR, "Expected END_FOR"); return target_bc.nextInstrOffset(); @@ -190,13 +199,9 @@ BCOffset BytecodeInstruction::nextInstrOffset() const { } _Py_CODEUNIT BytecodeInstruction::word() const { -#if PY_VERSION_HEX >= 0x030C0000 int opcode = unspecialize(uninstrumentedOpcode()); int oparg = _Py_OPARG(codeUnit(code_)[opcodeIndex().value()]); return _Py_MAKE_CODEUNIT(opcode, oparg); -#else - return codeUnit(code_)[opcodeIndex().value()]; -#endif } bool BytecodeInstruction::isAbsoluteControlFlow() const { @@ -212,8 +217,7 @@ bool BytecodeInstruction::isAbsoluteControlFlow() const { case POP_JUMP_IF_ZERO: case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: - // These instructions switched from absolute to relative in 3.11. - return PY_VERSION_HEX < 0x030B0000; + return false; default: return false; } @@ -227,9 +231,7 @@ BytecodeInstructionBlock::BytecodeInstructionBlock( BorrowedRef code, BCIndex start, BCIndex end) - : code_{ThreadedRef::create(code)}, - start_idx_{start}, - end_idx_{end} {} + : code_{code}, start_idx_{start}, end_idx_{end} {} BytecodeInstructionBlock::Iterator BytecodeInstructionBlock::begin() const { return Iterator{code_, start_idx_, end_idx_}; @@ -265,4 +267,4 @@ BorrowedRef BytecodeInstructionBlock::code() const { return code_; } -} // namespace jit +} // namespace cinderx::jit diff --git a/cinderx/Jit/bytecode.h b/cinderx/Jit/bytecode.h index 4830d638d..a13bcb1c5 100644 --- a/cinderx/Jit/bytecode.h +++ b/cinderx/Jit/bytecode.h @@ -9,11 +9,12 @@ #include "cinderx/Common/opcode_stubs.h" #include "cinderx/Interpreter/cinder_opcode.h" #include "cinderx/Jit/bytecode_offsets.h" +#include "cinderx/Jit/threaded_compile.h" #include #include -namespace jit { +namespace cinderx::jit { // A structured, immutable representation of a CPython bytecode. // @@ -46,6 +47,7 @@ class BytecodeInstruction { // Check if this instruction is a branch, a return, or a general basic block // terminator. bool isBranch() const; + bool isBackwardBranch() const; bool isReturn() const; bool isTerminator() const; @@ -92,6 +94,11 @@ class BytecodeInstruction { // // Extended args are handled automatically when iterating over the bytecode; // they will not appear in the stream of `BytecodeInstruction`s. +// +// BytecodeInstructionBlock borrows the code object - it does NOT keep it alive. +// Callers must ensure the code object outlives the block, which is true for +// all current uses (HIR building holds a Ref<> to the code +// object in Preloader / Function). class BytecodeInstructionBlock { public: explicit BytecodeInstructionBlock(BorrowedRef code); @@ -101,6 +108,14 @@ class BytecodeInstructionBlock { BCIndex start, BCIndex end); + ~BytecodeInstructionBlock() = default; + + BytecodeInstructionBlock(BytecodeInstructionBlock&&) = default; + BytecodeInstructionBlock& operator=(BytecodeInstructionBlock&&) = default; + + BytecodeInstructionBlock(const BytecodeInstructionBlock&) = delete; + BytecodeInstructionBlock& operator=(const BytecodeInstructionBlock&) = delete; + class Iterator { public: using iterator_category = std::input_iterator_tag; @@ -143,22 +158,6 @@ class BytecodeInstructionBlock { return bci_ == other.bci_; } - bool operator!=(const Iterator& other) const { - return !(*this == other); - } - - // Count the number of remaining bytecode indices in the block. - // - // This isn't useful in 3.11+ as instructions are variable length. So this - // doesn't tell you anything meaningful. Fortunately, we don't need it - // beyond 3.10. - Py_ssize_t remainingIndices() const { - if constexpr (PY_VERSION_HEX >= 0x030B0000) { - JIT_ABORT("remainingIndices() not supported in 3.11+"); - } - return end_idx_ - bci_.opcodeIndex() - 1; - } - private: BytecodeInstruction bci_; BCIndex end_idx_; @@ -182,7 +181,7 @@ class BytecodeInstructionBlock { BorrowedRef code() const; private: - ThreadedRef code_; + BorrowedRef code_; BCIndex start_idx_; BCIndex end_idx_; }; @@ -191,4 +190,4 @@ class BytecodeInstructionBlock { #define EXTENDED_OPCODE_FLAG 0 #endif -} // namespace jit +} // namespace cinderx::jit diff --git a/cinderx/Jit/bytecode_offsets.h b/cinderx/Jit/bytecode_offsets.h index b6b568768..05f1a7a56 100644 --- a/cinderx/Jit/bytecode_offsets.h +++ b/cinderx/Jit/bytecode_offsets.h @@ -16,7 +16,7 @@ #include #include -namespace jit { +namespace cinderx::jit { /* * BCOffsetBase is used to define two related types: BCOffset and BCIndex. @@ -193,32 +193,32 @@ inline _Py_CODEUNIT* operator+(_Py_CODEUNIT* code, BCIndex index) { return code + index.value(); } -inline std::ostream& operator<<(std::ostream& os, jit::BCOffset offset) { +inline std::ostream& operator<<(std::ostream& os, BCOffset offset) { return os << offset.value(); } -inline std::ostream& operator<<(std::ostream& os, jit::BCIndex index) { +inline std::ostream& operator<<(std::ostream& os, BCIndex index) { return os << index.value(); } -} // namespace jit +} // namespace cinderx::jit template <> -struct fmt::formatter : fmt::ostream_formatter {}; +struct fmt::formatter : fmt::ostream_formatter {}; template <> -struct fmt::formatter : fmt::ostream_formatter {}; +struct fmt::formatter : fmt::ostream_formatter {}; template <> -struct std::hash { - size_t operator()(const jit ::BCOffset& offset) const { +struct std::hash { + size_t operator()(const cinderx::jit::BCOffset& offset) const { return std::hash{}(offset.value()); } }; template <> -struct std::hash { - size_t operator()(const jit::BCIndex& index) const { +struct std::hash { + size_t operator()(const cinderx::jit::BCIndex& index) const { return std::hash{}(index.value()); } }; diff --git a/cinderx/Jit/cell_helpers.c b/cinderx/Jit/cell_helpers.c index a58bb93a1..021d657e5 100644 --- a/cinderx/Jit/cell_helpers.c +++ b/cinderx/Jit/cell_helpers.c @@ -13,11 +13,11 @@ #include "internal/pycore_cell.h" -PyObject* JITRT_LoadCellItem(PyCellObject* cell) { +PyObject* cx_load_cell_item(PyCellObject* cell) { return PyCell_GetRef(cell); } -PyObject* JITRT_SwapCellItem(PyCellObject* cell, PyObject* new_value) { +PyObject* cx_swap_cell_item(PyCellObject* cell, PyObject* new_value) { return PyCell_SwapTakeRef(cell, new_value); } diff --git a/cinderx/Jit/code_allocator.cpp b/cinderx/Jit/code_allocator.cpp index e6e1e2444..ef5a89c65 100644 --- a/cinderx/Jit/code_allocator.cpp +++ b/cinderx/Jit/code_allocator.cpp @@ -2,9 +2,12 @@ #include "cinderx/Jit/code_allocator.h" +#include "cinderx/Common/fork_support.h" #include "cinderx/Common/log.h" +#include "cinderx/Jit/codegen/code_section.h" #include "cinderx/Jit/config.h" -#include "cinderx/Jit/threaded_compile.h" +#include "cinderx/Jit/jit_rt.h" +#include "cinderx/module_state.h" #ifdef WIN32 #include @@ -13,9 +16,17 @@ #include #endif +#ifdef __APPLE__ +#include +#ifdef __aarch64__ +#include +#endif +#endif + #include +#include -namespace jit { +namespace cinderx::jit { using codegen::CodeSection; using codegen::codeSectionFromName; @@ -30,27 +41,26 @@ namespace { // 2MiB to match Linux's huge-page size. constexpr size_t kAllocSize = 1024 * 1024 * 2; -// Allocate memory for JIT'd code. -uint8_t* allocPages(size_t size) { -#ifndef WIN32 - void* res = mmap( - nullptr, - size, - PROT_EXEC | PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, - 0); - JIT_CHECK( - res != MAP_FAILED, - "Failed to allocate {} bytes of memory for code", - size); -#else - void* res = VirtualAlloc( - nullptr, size, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE); - JIT_CHECK( - res != nullptr, "Failed to allocate {} bytes of memory for code", size); +// On macOS ARM64, MAP_JIT memory requires toggling between writable and +// executable states per-thread via pthread_jit_write_protect_np. +void jitEnableWriting() { +#if defined(__APPLE__) && defined(__aarch64__) + pthread_jit_write_protect_np(0); +#endif +} + +void jitEnableExecuting( + [[maybe_unused]] void* addr, + [[maybe_unused]] size_t size, + [[maybe_unused]] void* cold_addr = nullptr, + [[maybe_unused]] size_t cold_size = 0) { +#if defined(__APPLE__) && defined(__aarch64__) + pthread_jit_write_protect_np(1); + sys_icache_invalidate(addr, size); + if (cold_size > 0) { + sys_icache_invalidate(cold_addr, cold_size); + } #endif - return static_cast(res); } bool setHugePages([[maybe_unused]] void* ptr, [[maybe_unused]] size_t size) { @@ -70,18 +80,142 @@ bool setHugePages([[maybe_unused]] void* ptr, [[maybe_unused]] size_t size) { return false; } +#if defined(__linux__) +// The linker script (instagram/server/native_python/linker_script.ld) reserves +// a region of address space immediately after .text for JIT code, delimited by +// these symbols. They are declared weak so that binaries built without the +// linker script (where the region doesn't exist) still link, with both symbols +// resolving to nullptr. +extern "C" { +extern char __cinder_jit_start[] __attribute__((weak)); +extern char __cinder_jit_end[] __attribute__((weak)); +} + +// Bump-allocator state for the linker-reserved __cinder_jit region. It's global +// process data so it's protected by cinder_jit_region_mutex_. +bool s_cinder_jit_region_checked = false; +std::atomic s_cinder_jit_cur = nullptr; +size_t s_cinder_jit_free = 0; +std::mutex cinder_jit_region_mutex_; + +// Prepare the linker-reserved __cinder_jit region for use. Called once on the +// first allocation. On success s_cinder_jit_cur points at the region with +// s_cinder_jit_free bytes remaining; on failure s_cinder_jit_cur stays nullptr +// and callers fall back to hinted allocation. +// +// The region comes from a `.cinder_jit (NOLOAD)` section in the linker script. +// Because that section is allocatable (SHF_ALLOC), the linker places it in a +// PT_LOAD segment, so the dynamic loader has *already mapped* this address +// range (as demand-zero anonymous pages) by the time we get here -- typically +// read-only, since the section carries no write/execute flags. We need to +// we upgrade the existing mapping's protection to RWX with mprotect. +void initCinderJitRegion() { + // Read the weak symbols into locals so the nullptr checks are on a pointer + // value (a weak undefined symbol resolves to nullptr) rather than the address + // of an array, which compilers would otherwise fold to always-true. + char* start = __cinder_jit_start; + char* end = __cinder_jit_end; + if (start == nullptr || end == nullptr || end <= start) { + // Binary wasn't linked with the linker script that reserves the region. + return; + } + + size_t region_size = static_cast(end - start); + + // The loader maps the region's PT_LOAD segment; make it writable and + // executable so we can emit and run JIT code from it. + if (mprotect(start, region_size, PROT_READ | PROT_WRITE | PROT_EXEC) != 0) { + JIT_LOG( + "Failed to mprotect cinder_jit region [{}, {}) as RWX, errno={}; " + "falling back to hinted allocation", + static_cast(start), + static_cast(end), + errno); + return; + } + + setHugePages(start, region_size); + + s_cinder_jit_cur.store( + reinterpret_cast(start), std::memory_order_relaxed); + s_cinder_jit_free = region_size; +} + +// Bump-allocate `size` bytes from the linker-reserved __cinder_jit region. +// Returns nullptr if the region is unavailable or exhausted, in which case the +// caller falls back to hinted allocation. The returned memory is already mapped +// PROT_READ | PROT_WRITE | PROT_EXEC. +uint8_t* allocFromCinderJitRegion(size_t size) { + if (s_cinder_jit_cur.load(std::memory_order_relaxed) == nullptr) { + return nullptr; + } + + std::lock_guard lock{cinder_jit_region_mutex_}; + if (size > s_cinder_jit_free) { + return nullptr; + } + uint8_t* res = s_cinder_jit_cur; + s_cinder_jit_cur.fetch_add(size, std::memory_order_relaxed); + s_cinder_jit_free -= size; + return res; +} +#endif // __linux__ + +// Allocate memory for JIT'd code. +uint8_t* allocPages(size_t size) { +#if defined(__linux__) + if (getConfig().hinted_code_allocation) { + // Prefer the linker-reserved region near .text when it is + // present. This region was reserved by the linker and it's up to the + // build system to reserve this near hot code. + if (uint8_t* region = allocFromCinderJitRegion(size); region != nullptr) { + return region; + } + } +#endif + +#ifndef WIN32 + int flags = MAP_PRIVATE | MAP_ANONYMOUS; +#ifdef __APPLE__ + flags |= MAP_JIT; +#endif + void* res = + mmap(nullptr, size, PROT_EXEC | PROT_READ | PROT_WRITE, flags, -1, 0); + JIT_CHECK( + res != MAP_FAILED, + "Failed to allocate {} bytes of memory for code", + size); +#else + void* res = VirtualAlloc( + nullptr, size, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE); + JIT_CHECK( + res != nullptr, "Failed to allocate {} bytes of memory for code", size); +#endif + return static_cast(res); +} + } // namespace +CodeAllocator::CodeAllocator() { +#if defined(__linux__) + std::lock_guard lock{cinder_jit_region_mutex_}; + if (!s_cinder_jit_region_checked) { + s_cinder_jit_region_checked = true; + initCinderJitRegion(); + } +#endif +} + ICodeAllocator* CodeAllocator::make() { - if (getConfig().multiple_code_sections) { - return new MultipleSectionCodeAllocator{}; - } else if (getConfig().use_huge_pages) { + if (getConfig().use_huge_pages) { return new CodeAllocatorCinder{}; } return new CodeAllocator{}; } AllocateResult CodeAllocator::addCode(asmjit::CodeHolder* code) { + std::lock_guard lock{runtime_mutex_}; + void* addr = nullptr; asmjit::Error error = runtime_.add(&addr, code); @@ -93,6 +227,8 @@ AllocateResult CodeAllocator::addCode(asmjit::CodeHolder* code) { } asmjit::Error CodeAllocator::releaseCode(void* code) { + std::lock_guard lock{runtime_mutex_}; + // Find the size of the allocated region. asmjit::JitAllocator* inner = runtime_.allocator(); asmjit::JitAllocator::Span span; @@ -100,6 +236,15 @@ asmjit::Error CodeAllocator::releaseCode(void* code) { return error; } + // The allocator may not actually free the code. Zero it out in debug builds + // so we know the memory is freed. + if constexpr (kDebug) { + auto rw = span.rw(); + if (rw != nullptr) { + memset(rw, 0, span.size()); + } + } + if (auto error = runtime_.release(code); error != asmjit::kErrorOk) { return error; } @@ -109,9 +254,12 @@ asmjit::Error CodeAllocator::releaseCode(void* code) { } bool CodeAllocator::contains(const void* ptr) const { + // query() is internally thread-safe, but it takes asmjit's own lock, which + // can't be recovered in a forked child. Going through runtime_mutex_ keeps + // that lock free whenever a fork can happen. + std::lock_guard lock{runtime_mutex_}; + asmjit::JitAllocator::Span unused; - // asmjit docs don't say that query() is thread-safe, but peeking at the - // implementation shows that it is. return runtime_.allocator()->query(unused, const_cast(ptr)) == asmjit::kErrorOk; } @@ -124,6 +272,18 @@ const asmjit::Environment& CodeAllocator::asmJitEnvironment() const { return runtime_.environment(); } +void CodeAllocator::atForkPrepare() { + runtime_mutex_.lock(); +} + +void CodeAllocator::atForkParent() { + runtime_mutex_.unlock(); +} + +void CodeAllocator::atForkChild() { + resetMutexAfterFork(runtime_mutex_); +} + CodeAllocatorCinder::~CodeAllocatorCinder() { for (std::span alloc : allocations_) { #ifndef WIN32 @@ -135,33 +295,175 @@ CodeAllocatorCinder::~CodeAllocatorCinder() { } } -AllocateResult CodeAllocatorCinder::addCode(asmjit::CodeHolder* code) { - ThreadedCompileSerialize guard; +void CodeAllocatorCinder::ensureSpace( + uint8_t*& alloc, + size_t& alloc_free, + size_t size, + bool use_huge_pages) { + if (alloc_free >= size) { + return; + } + + lost_bytes_.fetch_add(alloc_free, std::memory_order_relaxed); + + size_t chunk_size = ((size / kAllocSize) + 1) * kAllocSize; + uint8_t* res = allocPages(chunk_size); + if (use_huge_pages && setHugePages(res, chunk_size)) { + huge_allocs_.fetch_add(1, std::memory_order_relaxed); + } else { + fragmented_allocs_.fetch_add(1, std::memory_order_relaxed); + } + alloc = res; + allocations_.emplace_back(res, chunk_size); + alloc_free = chunk_size; +} + +void CodeAllocatorCinder::ensureSplitSpace( + size_t hot_needed, + size_t cold_needed) { + if (hot_alloc_free_ >= hot_needed && cold_alloc_free_ >= cold_needed) { + return; + } + + // When either side needs a new allocation, allocate a single contiguous + // region and split it. This guarantees hot and cold code are always within + // the same mmap region, so cross-section jumps stay within ARM64's relative + // branch range (±128MB for B/BL, ±1MB for B.cond). Without this, + // independent mmap() calls could place hot and cold regions too far apart, + // causing asmjit's relocateToBase()/resolveUnresolvedLinks() to fail with + // kErrorInvalidDisplacement. + lost_bytes_.fetch_add( + hot_alloc_free_ + cold_alloc_free_, std::memory_order_relaxed); + + size_t total_needed = hot_needed + cold_needed; + size_t chunk_size = ((total_needed / kAllocSize) + 1) * kAllocSize; + uint8_t* res = allocPages(chunk_size); + if (setHugePages(res, chunk_size)) { + huge_allocs_.fetch_add(1, std::memory_order_relaxed); + } else { + fragmented_allocs_.fetch_add(1, std::memory_order_relaxed); + } + allocations_.emplace_back(res, chunk_size); + + // Hot code grows forward from the start, cold code grows forward from a + // split point. Split proportionally so each side gets at least what it + // requested, distributing any surplus evenly. + size_t surplus = chunk_size - total_needed; + size_t hot_share = hot_needed + surplus / 2; + hot_alloc_ = res; + hot_alloc_free_ = hot_share; + cold_alloc_ = res + hot_share; + cold_alloc_free_ = chunk_size - hot_share; +} + +AllocateResult CodeAllocatorCinder::addSplitCode(asmjit::CodeHolder* code) { + size_t hot_size = 0; + size_t cold_size = 0; + + for (;;) { + // Compute how much space each section type needs. + hot_size = 0; + cold_size = 0; + for (asmjit::Section* section : code->sections()) { + CodeSection cs = codeSectionFromName(section->name()); + if (cs == CodeSection::kCold) { + cold_size += section->realSize(); + } else { + hot_size += section->realSize(); + } + } + + // Ensure we have enough space for both hot and cold code. +#if defined(__aarch64__) + // On ARM64, branch displacements are limited (±128MB for B/BL, ±1MB for + // B.cond). Allocate hot and cold from a single contiguous region so + // cross-section jumps are always in range. + ensureSplitSpace(hot_size, cold_size); +#else + // On x86-64, RIP-relative addressing has a ±2GB range which is large enough + // that independent allocations are unlikely to exceed it in practice. + ensureSpace(hot_alloc_, hot_alloc_free_, hot_size, true); + ensureSpace( + cold_alloc_, + cold_alloc_free_, + cold_size, + getConfig().cold_code_huge_pages); +#endif + + // Fix up offsets for each code section before resolving links. + // All offsets are relative to the hot allocation base so that asmjit can + // resolve cross-section jumps correctly. + size_t hot_offset = 0; + size_t cold_offset = static_cast(cold_alloc_ - hot_alloc_); + for (asmjit::Section* section : code->sections()) { + CodeSection cs = codeSectionFromName(section->name()); + if (cs == CodeSection::kCold) { + section->setOffset(cold_offset); + cold_offset += section->realSize(); + } else { + section->setOffset(hot_offset); + hot_offset += section->realSize(); + } + } + + bool changed = false; + PROPAGATE_ERROR(code->ensureBranchStubIslands(&changed)); + if (!changed) { + break; + } + } - PROPAGATE_ERROR(code->flatten()); PROPAGATE_ERROR(code->resolveUnresolvedLinks()); + PROPAGATE_ERROR(code->relocateToBase(uintptr_t(hot_alloc_))); - size_t max_code_size = code->codeSize(); - size_t alloc_size = ((max_code_size / kAllocSize) + 1) * kAllocSize; - if (current_alloc_free_ < max_code_size) { - lost_bytes_ += current_alloc_free_; + void* addr = hot_alloc_; + void* cold_addr = cold_alloc_; - uint8_t* res = allocPages(alloc_size); - if (!setHugePages(res, alloc_size)) { - fragmented_allocs_++; + // Copy each section's data to the appropriate allocation. + size_t total_size = 0; + jitEnableWriting(); + for (asmjit::Section* section : code->_sections) { + size_t buffer_size = section->bufferSize(); + if (buffer_size == 0) { + continue; + } + CodeSection cs = codeSectionFromName(section->name()); + if (cs == CodeSection::kCold) { + std::memcpy(cold_alloc_, section->data(), buffer_size); + cold_alloc_ += buffer_size; + cold_alloc_free_ -= buffer_size; } else { - huge_allocs_++; + std::memcpy(hot_alloc_, section->data(), buffer_size); + hot_alloc_ += buffer_size; + hot_alloc_free_ -= buffer_size; } - current_alloc_ = static_cast(res); - allocations_.emplace_back(res, alloc_size); - current_alloc_free_ = alloc_size; + total_size += buffer_size; + } + jitEnableExecuting(addr, hot_size, cold_addr, cold_size); + + used_bytes_.fetch_add(total_size, std::memory_order_relaxed); + return AllocateResult{addr, asmjit::kErrorOk}; +} + +AllocateResult CodeAllocatorCinder::addCode(asmjit::CodeHolder* code) { + std::lock_guard lock{allocator_mutex_}; + + if (getConfig().multiple_code_sections) { + return addSplitCode(code); } - PROPAGATE_ERROR(code->relocateToBase(uintptr_t(current_alloc_))); + PROPAGATE_ERROR(code->flatten()); + PROPAGATE_ERROR(code->resolveUnresolvedLinks()); + + size_t max_code_size = code->codeSize(); + ensureSpace(hot_alloc_, hot_alloc_free_, max_code_size, true); + + PROPAGATE_ERROR(code->relocateToBase(uintptr_t(hot_alloc_))); size_t actual_code_size = code->codeSize(); JIT_CHECK(actual_code_size <= max_code_size, "Code grew during relocation"); + jitEnableWriting(); for (asmjit::Section* section : code->_sections) { size_t offset = section->offset(); size_t buffer_size = section->bufferSize(); @@ -169,21 +471,22 @@ AllocateResult CodeAllocatorCinder::addCode(asmjit::CodeHolder* code) { JIT_CHECK( offset + buffer_size <= actual_code_size, "Inconsistent code size"); - std::memcpy(current_alloc_ + offset, section->data(), buffer_size); + std::memcpy(hot_alloc_ + offset, section->data(), buffer_size); if (virtual_size > buffer_size) { JIT_CHECK( offset + virtual_size <= actual_code_size, "Inconsistent code size"); std::memset( - current_alloc_ + offset + buffer_size, 0, virtual_size - buffer_size); + hot_alloc_ + offset + buffer_size, 0, virtual_size - buffer_size); } } - void* addr = current_alloc_; + void* addr = hot_alloc_; + jitEnableExecuting(addr, actual_code_size); - current_alloc_ += actual_code_size; - current_alloc_free_ -= actual_code_size; - used_bytes_ += actual_code_size; + hot_alloc_ += actual_code_size; + hot_alloc_free_ -= actual_code_size; + used_bytes_.fetch_add(actual_code_size, std::memory_order_relaxed); return AllocateResult{addr, asmjit::kErrorOk}; } @@ -194,7 +497,7 @@ asmjit::Error CodeAllocatorCinder::releaseCode([[maybe_unused]] void* code) { } bool CodeAllocatorCinder::contains(const void* ptr) const { - ThreadedCompileSerialize guard; + std::lock_guard lock{allocator_mutex_}; for (std::span alloc : allocations_) { if (alloc.data() <= ptr && ptr < alloc.data() + alloc.size()) { return true; @@ -203,139 +506,50 @@ bool CodeAllocatorCinder::contains(const void* ptr) const { return false; } -MultipleSectionCodeAllocator::~MultipleSectionCodeAllocator() { - if (code_alloc_ == nullptr) { - return; - } -#ifndef WIN32 - JIT_CHECK( - munmap(code_alloc_, total_allocation_size_) == 0, - "Freeing code sections failed"); -#else - VirtualFree(code_alloc_, 0, MEM_RELEASE); -#endif +void CodeAllocatorCinder::atForkPrepare() { + CodeAllocator::atForkPrepare(); + allocator_mutex_.lock(); } -/* - * At startup, we allocate a contiguous chunk of memory for all code sections - * equal to the sum of individual section sizes and subdivide internally. The - * code is contiguously allocated internally, but logically has pointers into - * each CodeSection. - */ -void MultipleSectionCodeAllocator::createSlabs() noexcept { - size_t hot_section_size = - asmjit::Support::alignUp(getConfig().hot_code_section_size, kAllocSize); - JIT_CHECK( - hot_section_size > 0, - "Hot code section must have non-zero size when using multiple sections."); - code_section_free_sizes_[CodeSection::kHot] = hot_section_size; - - size_t cold_section_size = getConfig().cold_code_section_size; - JIT_CHECK( - cold_section_size > 0, - "Cold code section must have non-zero size when using multiple " - "sections."); - code_section_free_sizes_[CodeSection::kCold] = cold_section_size; - - total_allocation_size_ = hot_section_size + cold_section_size; - - uint8_t* region = allocPages(total_allocation_size_); - setHugePages(region, hot_section_size); - - code_alloc_ = region; - code_sections_[CodeSection::kHot] = region; - region += hot_section_size; - code_sections_[CodeSection::kCold] = region; +void CodeAllocatorCinder::atForkParent() { + allocator_mutex_.unlock(); + CodeAllocator::atForkParent(); } -AllocateResult MultipleSectionCodeAllocator::addCode(asmjit::CodeHolder* code) { - ThreadedCompileSerialize guard; - - if (code_sections_.empty()) { - createSlabs(); - } - - size_t potential_code_size = code->codeSize(); - used_bytes_ += potential_code_size; - // We fall back to the default size of code allocation if the - // code doesn't fit into either section, and we can make this check more - // granular by comparing sizes section-by-section. - if (code_section_free_sizes_[CodeSection::kHot] < potential_code_size || - code_section_free_sizes_[CodeSection::kCold] < potential_code_size) { - JIT_LOG( - "Not enough memory to split code across sections, falling back to " - "normal allocation."); - void* addr = nullptr; - asmjit::Error err = runtime_.add(&addr, code); - return AllocateResult{addr, err}; - } - - // Fix up the offsets for each code section before resolving links. - // Both the `.text` and `.addrtab` sections are written to the hot section, - // and we need to resolve offsets between them properly. - // In order to properly keep track of multiple text sections corresponding to - // the same physical section to allocate to, we keep a map from - // section->offset from start of hot section. - std::unordered_map offsets; - offsets[CodeSection::kHot] = 0; - offsets[CodeSection::kCold] = - code_sections_[CodeSection::kCold] - code_sections_[CodeSection::kHot]; - - for (asmjit::Section* section : code->sections()) { - CodeSection code_section = codeSectionFromName(section->name()); - uint64_t offset = offsets[code_section]; - uint64_t realSize = section->realSize(); - section->setOffset(offset); - // Since all sections lie on a contiguous slab, we rely on setting the - // offsets of sections to allow AsmJit to properly resolve links across - // different sections (offset 0 being the start of the hot code section). - offsets[code_section] = offset + realSize; - } - - // Assuming that the offsets are set properly, relocating all code to be - // relative to the start of the hot code will ensure jumps are correct. - PROPAGATE_ERROR(code->resolveUnresolvedLinks()); - PROPAGATE_ERROR( - code->relocateToBase(uintptr_t(code_sections_[CodeSection::kHot]))); - - // We assume that the hot section of the code is non-empty. This would be - // incorrect for a completely cold function. - JIT_CHECK( - code->textSection()->realSize() > 0, - "Every function must have a non-empty hot section."); - void* addr = code_sections_[CodeSection::kHot]; +void CodeAllocatorCinder::atForkChild() { + resetMutexAfterFork(allocator_mutex_); + CodeAllocator::atForkChild(); +} - for (asmjit::Section* section : code->_sections) { - size_t buffer_size = section->bufferSize(); - // Might not have generated any cold code. - if (buffer_size == 0) { - continue; - } - CodeSection code_section = codeSectionFromName(section->name()); - code_section_free_sizes_[code_section] -= buffer_size; - std::memcpy(code_sections_[code_section], section->data(), buffer_size); - code_sections_[code_section] += buffer_size; +void codeAllocatorAtForkPrepare() { + ModuleState* state = cinderx::getModuleState(); + if (state != nullptr && state->code_allocator != nullptr) { + state->code_allocator->atForkPrepare(); } - - return AllocateResult{addr, asmjit::kErrorOk}; +#if defined(__linux__) + // Innermost: ensureSpace() reaches this while holding allocator_mutex_. + cinder_jit_region_mutex_.lock(); +#endif } -asmjit::Error MultipleSectionCodeAllocator::releaseCode( - [[maybe_unused]] void* code) { - // TODO(T233607793): Actually implement deallocating memory. - return asmjit::kErrorOk; +void codeAllocatorAtForkParent() { +#if defined(__linux__) + cinder_jit_region_mutex_.unlock(); +#endif + ModuleState* state = cinderx::getModuleState(); + if (state != nullptr && state->code_allocator != nullptr) { + state->code_allocator->atForkParent(); + } } -bool MultipleSectionCodeAllocator::contains(const void* ptr) const { - // Have to check both the hot/cold slab and the asmjit allocator. The latter - // is already thread-safe. - { - ThreadedCompileSerialize guard; - if (code_alloc_ <= ptr && ptr < code_alloc_ + total_allocation_size_) { - return true; - } +void codeAllocatorAtForkChild() { +#if defined(__linux__) + new (&cinder_jit_region_mutex_) std::mutex{}; +#endif + ModuleState* state = cinderx::getModuleState(); + if (state != nullptr && state->code_allocator != nullptr) { + state->code_allocator->atForkChild(); } - return CodeAllocator::contains(ptr); } -} // namespace jit +} // namespace cinderx::jit diff --git a/cinderx/Jit/code_allocator.h b/cinderx/Jit/code_allocator.h index 4c0ee461e..edfce4f1c 100644 --- a/cinderx/Jit/code_allocator.h +++ b/cinderx/Jit/code_allocator.h @@ -3,16 +3,15 @@ #pragma once #include "cinderx/Jit/code_allocator_iface.h" -#include "cinderx/Jit/codegen/code_section.h" #include #include +#include #include -#include #include -namespace jit { +namespace cinderx::jit { /* A CodeAllocator allocates memory for live JIT code. This is an abstract @@ -29,11 +28,12 @@ namespace jit { */ class CodeAllocator : public ICodeAllocator { public: + CodeAllocator(); ~CodeAllocator() override = default; // To be called once by JIT initialization after enough configuration has been // loaded to determine which global code allocator type to use. - static ICodeAllocator* make(); + [[nodiscard]] static ICodeAllocator* make(); AllocateResult addCode(asmjit::CodeHolder* code) override; asmjit::Error releaseCode(void* code) override; @@ -41,71 +41,103 @@ class CodeAllocator : public ICodeAllocator { size_t usedBytes() const override; const asmjit::Environment& asmJitEnvironment() const override; + void atForkPrepare() override; + void atForkParent() override; + void atForkChild() override; + protected: asmjit::JitRuntime runtime_; std::atomic used_bytes_{0}; + + // Serializes every operation that reaches asmjit's own JitAllocator lock. + // That lock is private to asmjit with no way to reset it in a forked child, + // so this makes it observably free at fork time instead: atForkPrepare() + // holds this, which means no thread can be inside asmjit when the fork + // happens. + mutable std::mutex runtime_mutex_; }; // A code allocator which tries to allocate all code on huge pages. +// +// When multiple code sections are enabled, hot code is allocated on huge pages +// and cold code is allocated on separate pages (optionally huge pages as well, +// controlled by the cold_code_huge_pages config). class CodeAllocatorCinder : public CodeAllocator { public: ~CodeAllocatorCinder() override; size_t lostBytes() const { - return lost_bytes_; + return lost_bytes_.load(std::memory_order_relaxed); } size_t fragmentedAllocs() const { - return fragmented_allocs_; + return fragmented_allocs_.load(std::memory_order_relaxed); } size_t hugeAllocs() const { - return huge_allocs_; + return huge_allocs_.load(std::memory_order_relaxed); } AllocateResult addCode(asmjit::CodeHolder* code) override; asmjit::Error releaseCode(void* code) override; bool contains(const void* ptr) const override; + void atForkPrepare() override; + void atForkParent() override; + void atForkChild() override; + private: - // List of chunks allocated for use in deallocation + // Add code with hot/cold section splitting. Called by addCode() when + // multiple_code_sections is enabled. Caller must hold allocator_mutex_. + AllocateResult addSplitCode(asmjit::CodeHolder* code); + + // Ensure the given bump allocator has at least `size` bytes free, allocating + // a new chunk if necessary. + void ensureSpace( + uint8_t*& alloc, + size_t& alloc_free, + size_t size, + bool use_huge_pages); + + // Ensure both hot and cold bump allocators have enough space for the given + // sizes. If either needs a new allocation, a single contiguous region is + // allocated and split between hot (first half) and cold (second half). This + // guarantees cross-section jumps are within ARM64's relative branch range + // (±128MB for B/BL, ±1MB for B.cond). Used only on aarch64. + void ensureSplitSpace(size_t hot_needed, size_t cold_needed); + + // Protects all allocator-owned state used by addCode()/contains(). + mutable std::mutex allocator_mutex_; + + // List of all chunks allocated, for use in deallocation and contains(). std::vector> allocations_; - // Pointer to next free address in the current chunk - uint8_t* current_alloc_{nullptr}; - // Free space in the current chunk - size_t current_alloc_free_{0}; + // Hot code allocation state. + uint8_t* hot_alloc_{nullptr}; + size_t hot_alloc_free_{0}; + + // Cold code allocation state (used when multiple code sections are enabled). + uint8_t* cold_alloc_{nullptr}; + size_t cold_alloc_free_{0}; // Number of bytes in total lost when allocations didn't fit neatly into // the bytes remaining in a chunk so a new one was allocated. - size_t lost_bytes_{0}; - // Number of chunks allocated (= to number of huge pages used) - size_t huge_allocs_{0}; + std::atomic lost_bytes_{0}; + // Number of chunks allocated which successfully used huge pages. + std::atomic huge_allocs_{0}; // Number of chunks allocated which did not use huge pages. - size_t fragmented_allocs_{0}; + std::atomic fragmented_allocs_{0}; }; -class MultipleSectionCodeAllocator : public CodeAllocator { - public: - ~MultipleSectionCodeAllocator() override; - - AllocateResult addCode(asmjit::CodeHolder* code) override; - asmjit::Error releaseCode(void* code) override; - bool contains(const void* ptr) const override; - - private: - void createSlabs() noexcept; - - std::unordered_map code_sections_; - std::unordered_map code_section_free_sizes_; - - uint8_t* code_alloc_{nullptr}; - size_t total_allocation_size_{0}; -}; +// pthread_atfork() handlers covering both the process-global code-allocation +// state and the current ICodeAllocator, if one exists. +void codeAllocatorAtForkPrepare(); +void codeAllocatorAtForkParent(); +void codeAllocatorAtForkChild(); void populateCodeSections( std::vector>& output_vector, asmjit::CodeHolder& code, void* entry); -} // namespace jit +} // namespace cinderx::jit diff --git a/cinderx/Jit/code_allocator_iface.h b/cinderx/Jit/code_allocator_iface.h index 1d2b1fd2d..1482aa435 100644 --- a/cinderx/Jit/code_allocator_iface.h +++ b/cinderx/Jit/code_allocator_iface.h @@ -6,7 +6,7 @@ #include -namespace jit { +namespace cinderx::jit { // Resulting address and status code after calling ICodeAllocator::addCode(). struct AllocateResult { @@ -34,6 +34,14 @@ class ICodeAllocator { // Get the asmjit environment used by this allocator. virtual const asmjit::Environment& asmJitEnvironment() const = 0; + + // pthread_atfork() handlers, called via codeAllocatorAtFork*(). Compile + // threads allocate code with the GIL released, so a child forked at the + // wrong moment would otherwise inherit an allocator lock held by a thread + // that no longer exists. + virtual void atForkPrepare() {} + virtual void atForkParent() {} + virtual void atForkChild() {} }; -} // namespace jit +} // namespace cinderx::jit diff --git a/cinderx/Jit/code_patcher.cpp b/cinderx/Jit/code_patcher.cpp index 49bdee2d9..8d066b43d 100644 --- a/cinderx/Jit/code_patcher.cpp +++ b/cinderx/Jit/code_patcher.cpp @@ -2,20 +2,16 @@ #include "cinderx/Jit/code_patcher.h" +#include "cinderx/Common/define.h" #include "cinderx/Common/log.h" #include "cinderx/Common/util.h" -#include "cinderx/Jit/codegen/arch/detection.h" +#include #include -#include -#ifdef Py_GIL_DISABLED #include -#ifdef CINDER_X86_64 -#include -#endif -#endif +#include -namespace jit { +namespace cinderx::jit { namespace { @@ -37,9 +33,9 @@ CINDER_UNSUPPORTED constexpr auto kJmpNopBytes = std::to_array({0x00}); #endif -// Compute an unsigned 32-bit jump displacement. -uint32_t jumpDisplacement(uintptr_t from, uintptr_t to) { - auto disp = to - from; +// Compute a signed 32-bit jump displacement. +int32_t jumpDisplacement(uintptr_t from, uintptr_t to) { + auto disp = static_cast(to - from); #if defined(CINDER_X86_64) disp -= kJmpNopBytes.size(); @@ -50,13 +46,14 @@ uint32_t jumpDisplacement(uintptr_t from, uintptr_t to) { "Can't encode jump from {:#x} to {:#x} as relative", from, to); - return static_cast(disp); + return static_cast(disp); } // Given the starting address and displacement operand of a jump instruction, // resolve it to a target address. -uintptr_t resolveDisplacement(uintptr_t from, uint32_t displacement) { - auto disp = from + displacement; +uintptr_t resolveDisplacement(uintptr_t from, int32_t displacement) { + auto disp = + from + static_cast(static_cast(displacement)); #if defined(CINDER_X86_64) disp += kJmpNopBytes.size(); @@ -122,49 +119,50 @@ std::span CodePatcher::storedBytes() const { } void CodePatcher::swap() { -#ifdef Py_GIL_DISABLED SwapLockGuard lock{*this}; -#endif -#if defined(CINDER_X86_64) && defined(Py_GIL_DISABLED) // On x86 the patchpoint is up to 7 bytes (aligned to 8 bytes by the code // generator). However, we work with 8 bytes here as that should be an // atomically writable size on x86. - static_assert(sizeof(uint64_t) >= sizeof(data_)); - static_assert(std::atomic_ref::is_always_lock_free == true); - JIT_CHECK( - reinterpret_cast(patchpoint_) % 8 == 0, "Not 8-byte aligned"); - uint64_t qword; - std::memcpy(&qword, patchpoint_, sizeof(qword)); - - auto* qword_bytes = reinterpret_cast(&qword); - std::swap_ranges(qword_bytes, qword_bytes + flags_.data_len, data_.data()); - - std::atomic_ref{*reinterpret_cast(patchpoint_)}.store( - qword, std::memory_order_relaxed); -#else - decltype(data_) temp; - std::memcpy(temp.data(), patchpoint_, flags_.data_len); - std::memcpy(patchpoint_, data_.data(), flags_.data_len); - std::memcpy(data_.data(), temp.data(), flags_.data_len); -#endif + if constexpr (kFreeThreadedBuild && kBuildArch == Arch::kX86_64) { + static_assert(sizeof(uint64_t) >= sizeof(data_)); + static_assert(std::atomic_ref::is_always_lock_free == true); + JIT_CHECK( + reinterpret_cast(patchpoint_) % 8 == 0, + "Not 8-byte aligned"); + uint64_t qword; + std::memcpy(&qword, patchpoint_, sizeof(qword)); + + auto* qword_bytes = reinterpret_cast(&qword); + std::swap_ranges(qword_bytes, qword_bytes + flags_.data_len, data_.data()); + + std::atomic_ref{*reinterpret_cast(patchpoint_)}.store( + qword, std::memory_order_relaxed); + } else { + decltype(data_) temp; + std::memcpy(temp.data(), patchpoint_, flags_.data_len); + std::memcpy(patchpoint_, data_.data(), flags_.data_len); + std::memcpy(data_.data(), temp.data(), flags_.data_len); + } -#ifdef Py_GIL_DISABLED - // Flush CPU caches, including the instruction cache, so all cores will see - // the update. Note for x86 this is a no-op as caches are coherent. - __builtin___clear_cache( - reinterpret_cast(patchpoint_), - reinterpret_cast(patchpoint_) + flags_.data_len); -#endif + if constexpr (kFreeThreadedBuild) { + // Flush CPU caches, including the instruction cache, so all cores will see + // the update. Note for x86 this is a no-op as caches are coherent. + __builtin___clear_cache( + reinterpret_cast(patchpoint_), + reinterpret_cast(patchpoint_) + flags_.data_len); + } } -#ifdef Py_GIL_DISABLED // We use a custom spin-lock implementation as I'm not aware of a generic way of // implementing a lock where the mutex is bit-packed with other data. This // should be fine as the critical section is a short, slow-path, and should // only happen very rarely. CodePatcher::SwapLockGuard::SwapLockGuard(CodePatcher& patcher) : patcher_(patcher) { + if constexpr (!kFreeThreadedBuild) { + return; + } std::atomic_ref ref{patcher_.flags_byte_}; while (true) { uint8_t expected = ref.load(std::memory_order_relaxed); @@ -180,10 +178,12 @@ CodePatcher::SwapLockGuard::SwapLockGuard(CodePatcher& patcher) } CodePatcher::SwapLockGuard::~SwapLockGuard() { + if constexpr (!kFreeThreadedBuild) { + return; + } std::atomic_ref{patcher_.flags_byte_}.fetch_and( static_cast(~lockBit()), std::memory_order_release); } -#endif JumpPatcher::JumpPatcher() { // Initializes to a nop. @@ -198,7 +198,7 @@ void JumpPatcher::linkJump(uintptr_t patchpoint, uintptr_t jump_target) { #if defined(CINDER_X86_64) // 32 bit relative jump - https://www.felixcloutier.com/x86/jmp buf[0] = 0xe9; - std::memcpy(buf.data() + 1, &disp, sizeof(uint32_t)); + std::memcpy(buf.data() + 1, &disp, sizeof(disp)); #elif defined(CINDER_AARCH64) JIT_CHECK( disp % 4 == 0, "Jump displacement must be a multiple of 4, got {}", disp); @@ -206,7 +206,7 @@ void JumpPatcher::linkJump(uintptr_t patchpoint, uintptr_t jump_target) { disp /= 4; JIT_CHECK(fitsSignedInt<26>(disp), "Not enough bits to encode relative jump"); - uint32_t insn = 0x14000000 | disp; + uint32_t insn = 0x14000000 | (static_cast(disp) & 0x03ffffff); std::memcpy(buf.data(), &insn, sizeof(uint32_t)); #else (void)disp; @@ -226,16 +226,15 @@ uint8_t* JumpPatcher::jumpTarget() const { "Must have linked a {}-byte jump instruction into a JumpPatcher", kJmpNopBytes.size()); - uint32_t disp = 0; + int32_t disp = 0; #if defined(CINDER_X86_64) std::memcpy(&disp, bytes.data() + 1, bytes.size() - 1); #elif defined(CINDER_AARCH64) - std::memcpy(&disp, bytes.data(), bytes.size()); - disp &= 0x03ffffff; // extract out 26-bit immediate - if (disp & 0x02000000) { - disp |= 0xfc000000; // sign-extend 26-bit immediate to 32-bit displacement - } + uint32_t insn = 0; + std::memcpy(&insn, bytes.data(), bytes.size()); + // Extract 26-bit immediate and sign-extend to 32 bits. + disp = static_cast(insn << 6) >> 6; disp *= 4; #else CINDER_UNSUPPORTED @@ -245,4 +244,4 @@ uint8_t* JumpPatcher::jumpTarget() const { resolveDisplacement(reinterpret_cast(patchpoint_), disp)); } -} // namespace jit +} // namespace cinderx::jit diff --git a/cinderx/Jit/code_patcher.h b/cinderx/Jit/code_patcher.h index 068f0070c..c29879a58 100644 --- a/cinderx/Jit/code_patcher.h +++ b/cinderx/Jit/code_patcher.h @@ -7,7 +7,7 @@ #include #include -namespace jit { +namespace cinderx::jit { // A CodePatcher is used by the runtime to overwrite parts of compiled code. // Often times this is used to patch in a jump to a deopt exit when an invariant @@ -106,6 +106,7 @@ class CodePatcher { uint8_t data_len : 3; bool is_patched : 1; bool lock : 1; + uint8_t unused : 3; }; union { Flags flags_{}; @@ -116,6 +117,7 @@ class CodePatcher { static constexpr uint8_t lockBit() { Flags f{}; f.lock = true; + static_assert(std::has_unique_object_representations_v); return std::bit_cast(f); } }; @@ -136,4 +138,4 @@ class JumpPatcher : public CodePatcher { uint8_t* jumpTarget() const; }; -} // namespace jit +} // namespace cinderx::jit diff --git a/cinderx/Jit/code_runtime.cpp b/cinderx/Jit/code_runtime.cpp index 3bce8ba1e..5a980bb53 100644 --- a/cinderx/Jit/code_runtime.cpp +++ b/cinderx/Jit/code_runtime.cpp @@ -3,8 +3,9 @@ #include "cinderx/Jit/code_runtime.h" #include "cinderx/Common/util.h" +#include "cinderx/Jit/threaded_compile.h" -namespace jit { +namespace cinderx::jit { GenYieldPoint::GenYieldPoint(std::size_t deopt_idx, ptrdiff_t yield_from_offset) : deopt_idx_{deopt_idx}, yield_from_offset_{yield_from_offset} {} @@ -29,26 +30,22 @@ ptrdiff_t GenYieldPoint::yieldFromOffset() const { return yield_from_offset_; } -bool RuntimeFrameState::isGen() const { +bool CodeRuntime::isGen() const { return code()->co_flags & kCoFlagsAnyGenerator; } -BorrowedRef RuntimeFrameState::code() const { +BorrowedRef CodeRuntime::code() const { return code_; } -BorrowedRef RuntimeFrameState::builtins() const { +BorrowedRef CodeRuntime::builtins() const { return builtins_; } -BorrowedRef RuntimeFrameState::globals() const { +BorrowedRef CodeRuntime::globals() const { return globals_; } -BorrowedRef RuntimeFrameState::func() const { - return func_; -} - CodeRuntime::CodeRuntime(BorrowedRef func) : CodeRuntime{ BorrowedRef{func->func_code}, @@ -59,27 +56,40 @@ CodeRuntime::CodeRuntime( BorrowedRef code, BorrowedRef builtins, BorrowedRef globals) - : frame_state_{code, builtins, globals} { - // Ensure code, globals, and builtins objects live as long as their compiled - // functions. - addReference(code); - addReference(builtins); - addReference(globals); -} + : code_{code}, builtins_{builtins}, globals_{globals} {} void CodeRuntime::addReference(BorrowedRef<> obj) { - // Serialize as we modify the ref-count to obj which may be widely accessible. - ThreadedCompileSerialize guard; - references_.emplace(ThreadedRef<>::create(obj)); + JIT_DCHECK( + ThreadedCompileContext::canAccessSharedData(), "lock should be held"); + if (!_Py_IsImmortal(obj)) { + references_.emplace(Ref<>::create(obj)); + } +} + +void CodeRuntime::transferReferences(std::unordered_set>&& refs) { + JIT_DCHECK( + ThreadedCompileContext::canAccessSharedData(), "lock should be held"); + references_.merge(std::move(refs)); } void CodeRuntime::releaseReferences() { - // Serialize as we modify ref-counts which may be widely accessible. - ThreadedCompileSerialize guard; - references_.clear(); + // We want to be careful here with the freeing of these references. Freeing + // the objects could cause our CompiledFunction to be freed as well so first + // we grab the references and then clear them. + JIT_DCHECK( + ThreadedCompileContext::canAccessSharedData(), "lock should be held"); + + std::unordered_set> refs; #if PY_VERSION_HEX >= 0x030E0000 && defined(ENABLE_LIGHTWEIGHT_FRAMES) - reifier_.reset(nullptr); + Ref<> tmp; #endif + { + refs = std::move(references_); +#if PY_VERSION_HEX >= 0x030E0000 && defined(ENABLE_LIGHTWEIGHT_FRAMES) + reifier_.reset(nullptr); +#endif + } + // and then we let the dtors clean everything up } GenYieldPoint* CodeRuntime::addGenYieldPoint(GenYieldPoint&& gen_yield_point) { @@ -87,7 +97,7 @@ GenYieldPoint* CodeRuntime::addGenYieldPoint(GenYieldPoint&& gen_yield_point) { return &gen_yield_points_.back(); } -std::size_t CodeRuntime::addDeoptMetadata(DeoptMetadata&& deopt_meta) { +std::size_t CodeRuntime::addRawDeoptMetadata(DeoptMetadata&& deopt_meta) { deopt_metadatas_.emplace_back(std::move(deopt_meta)); return deopt_metadatas_.size() - 1; } @@ -104,10 +114,6 @@ const std::vector& CodeRuntime::deoptMetadatas() const { return deopt_metadatas_; } -const RuntimeFrameState* CodeRuntime::frameState() const { - return &frame_state_; -} - int CodeRuntime::frameSize() const { return frame_size_; } @@ -116,8 +122,88 @@ void CodeRuntime::setFrameSize(int size) { frame_size_ = size; } +uint32_t CodeRuntime::spillWords() const { + return spill_words_; +} + +void CodeRuntime::setSpillWords(uint32_t words) { + spill_words_ = words; +} + +GenResumeFunc CodeRuntime::genResumeEntry() const { + return gen_resume_entry_; +} + +void CodeRuntime::setGenResumeEntry(GenResumeFunc resume_entry) { + gen_resume_entry_ = resume_entry; +} + DebugInfo* CodeRuntime::debugInfo() { return &debug_info_; } -} // namespace jit +void** CodeRuntime::allocateTypeCheckJumpTable(size_t num_entries) { + type_check_jump_table_ = std::make_unique(num_entries); + return type_check_jump_table_.get(); +} + +bool CodeRuntime::isCleared() const { + // We always add some references when we first create the CodeRuntime, so we + // know if no references are left we've been cleared. + return references_.empty(); +} + +int CodeRuntime::traverse(visitproc visit, void* arg) { + // Only traverse objects that this CodeRuntime owns strong references to. + // The references_ set holds strong references. + // code_, builtins_, globals_ are BorrowedRef pointing to the same objects + // already in references_ - don't double-count. + for (const auto& ref : references_) { + Py_VISIT(ref.get()); + } + if (auto ref = reifier()) { + Py_VISIT(ref.get()); + } + + return 0; +} + +std::optional CodeRuntime::getCallsiteDeoptExit( + uintptr_t return_addr) const { + auto it = callsite_deopt_exits_.find(return_addr); + if (it != callsite_deopt_exits_.end()) { + return it->second; + } + return std::nullopt; +} + +void CodeRuntime::addCallsiteDeoptExit( + uintptr_t return_addr, + uintptr_t deopt_exit_addr) { + callsite_deopt_exits_[return_addr] = deopt_exit_addr; +} + +void CodeRuntime::setReifier([[maybe_unused]] Ref<>&& reifier) { +#if PY_VERSION_HEX >= 0x030E0000 && defined(ENABLE_LIGHTWEIGHT_FRAMES) + reifier_ = std::move(reifier); +#endif +} + +BorrowedRef<> CodeRuntime::reifier() { +#if PY_VERSION_HEX >= 0x030E0000 && defined(ENABLE_LIGHTWEIGHT_FRAMES) + return reifier_; +#else + return nullptr; +#endif +} + +void CodeRuntime::setCompiledFunction( + BorrowedRef compiled_func) { + compiled_function_ = compiled_func; +} + +BorrowedRef CodeRuntime::compiledFunction() const { + return compiled_function_; +} + +} // namespace cinderx::jit diff --git a/cinderx/Jit/code_runtime.h b/cinderx/Jit/code_runtime.h index bee08f656..5f0dd2bf1 100644 --- a/cinderx/Jit/code_runtime.h +++ b/cinderx/Jit/code_runtime.h @@ -3,15 +3,20 @@ #pragma once #include "cinderx/Common/ref.h" +#include "cinderx/Common/util.h" #include "cinderx/Jit/debug_info.h" #include "cinderx/Jit/deopt.h" #include "cinderx/Jit/threaded_compile.h" #include #include +#include +#include #include -namespace jit { +namespace cinderx::jit { + +class CompiledFunction; constexpr ptrdiff_t kInvalidYieldFromOffset = std::numeric_limits::max(); @@ -39,80 +44,34 @@ class GenYieldPoint { const ptrdiff_t yield_from_offset_; }; -class alignas(16) RuntimeFrameState { - public: - static constexpr int64_t codeOffset() { - return offsetof(RuntimeFrameState, code_); - } - - RuntimeFrameState( - BorrowedRef code, - BorrowedRef builtins, - BorrowedRef globals, - BorrowedRef func = nullptr) - : code_{code}, builtins_{builtins}, globals_{globals}, func_{func} {} - - // Check if this is a generator frame. - bool isGen() const; - - BorrowedRef code() const; - BorrowedRef builtins() const; - BorrowedRef globals() const; - BorrowedRef func() const; - - private: - // All fields are owned by the CodeRuntime that owns this RuntimeFrameState. - - BorrowedRef code_; - BorrowedRef builtins_; - BorrowedRef globals_; - // The function is only set for inlined frames. - BorrowedRef func_; -}; - // Runtime data for a PyCodeObject object, containing caches and any other data // associated with a JIT-compiled function. class alignas(16) CodeRuntime { public: - static constexpr int64_t frameStateOffset() { -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Winvalid-offsetof" - return offsetof(CodeRuntime, frame_state_); -#pragma GCC diagnostic pop - } - - static constexpr int64_t codeOffset() { - return CodeRuntime::frameStateOffset() + RuntimeFrameState::codeOffset(); - } - explicit CodeRuntime(BorrowedRef func); CodeRuntime( BorrowedRef code, BorrowedRef builtins, BorrowedRef globals); - template - RuntimeFrameState* allocateRuntimeFrameState(Args&&... args) { - return inlined_frame_states_ - .emplace_back( - std::make_unique(std::forward(args)...)) - .get(); - } - // Ensure that this CodeRuntime owns a reference to the given borrowed // object, keeping it alive for use by the compiled code. Make CodeRuntime a // new owner of the object. void addReference(BorrowedRef<> obj); + // Transfer references to be owned by the CodeRuntime. + void transferReferences(std::unordered_set>&& refs); + // Release any references this CodeRuntime holds to Python objects. void releaseReferences(); // Store meta-data about generator yield point. GenYieldPoint* addGenYieldPoint(GenYieldPoint&& gen_yield_point); - // Add metadata used during a deopt. Return an ID that can be used to fetch - // the metadata from generated code. - std::size_t addDeoptMetadata(DeoptMetadata&& deopt_meta); + // Add raw deopt metadata that was constructed without a DeoptBase, such as + // callsite live-value metadata for helper calls. This bypasses the + // instruction-based dedup cache. + std::size_t addRawDeoptMetadata(DeoptMetadata&& deopt_meta); // Get a reference to the DeoptMetadata with the given ID. DeoptMetadata& getDeoptMetadata(std::size_t id); @@ -121,34 +80,59 @@ class alignas(16) CodeRuntime { // Get all deopt metadatas for the given CodeRuntime. const std::vector& deoptMetadatas() const; - // Get the top-level runtime frame state for this CodeRuntime's PyCodeObject. - const RuntimeFrameState* frameState() const; + // Check if this is a generator/coroutine/async generator. + bool isGen() const; + + BorrowedRef code() const; + BorrowedRef builtins() const; + BorrowedRef globals() const; // Get and set the total size of a stack frame for this compiled code object. int frameSize() const; void setFrameSize(int size); + // Get and set the number of spill words for generators. + uint32_t spillWords() const; + void setSpillWords(uint32_t words); + + // Get and set the address a generator resumes execution at. Only meaningful + // for generators, and only resolvable once code generation has bound the + // resume entry label to an address. + GenResumeFunc genResumeEntry() const; + void setGenResumeEntry(GenResumeFunc resume_entry); + DebugInfo* debugInfo(); -#if PY_VERSION_HEX >= 0x030E0000 && defined(ENABLE_LIGHTWEIGHT_FRAMES) - void setReifier(BorrowedRef<> reifier) { - ThreadedCompileSerialize guard; - reifier_ = ThreadedRef<>::create(reifier); - } - BorrowedRef<> reifier() { - return reifier_; - } -#else - BorrowedRef<> reifier() { - return nullptr; - } -#endif + // Allocate a jump table for static type check dispatch. + // Returns a pointer to the table data (valid for the lifetime of this + // CodeRuntime). + void** allocateTypeCheckJumpTable(size_t num_entries); + + // Traverse all GC-reachable objects held by this CodeRuntime. + int traverse(visitproc visit, void* arg); + + // True if the references have been cleared + bool isCleared() const; + + std::optional getCallsiteDeoptExit(uintptr_t return_addr) const; + + void addCallsiteDeoptExit(uintptr_t return_addr, uintptr_t deopt_exit_addr); + + void setReifier(Ref<>&& reifier); + + BorrowedRef<> reifier(); + + void setCompiledFunction(BorrowedRef compiled_func); + + BorrowedRef compiledFunction() const; + private: - RuntimeFrameState frame_state_; - std::vector> inlined_frame_states_; + BorrowedRef code_; + BorrowedRef builtins_; + BorrowedRef globals_; // References owned by this CodeRuntime. - std::unordered_set> references_; + std::unordered_set> references_; // Metadata about yield points. Deque so we can have raw pointers to content. std::deque gen_yield_points_; @@ -158,11 +142,26 @@ class alignas(16) CodeRuntime { std::vector deopt_metadatas_; #if PY_VERSION_HEX >= 0x030E0000 && defined(ENABLE_LIGHTWEIGHT_FRAMES) - ThreadedRef<> reifier_; + Ref<> reifier_; #endif + // Jump table for static type check dispatch (indexed by defaulted_arg_count). + // Entries are resolved to code addresses after code generation. + std::unique_ptr type_check_jump_table_; + + // Map from call return addresses to post-call guard deopt exits. + // Built during codegen, used by deoptAllJitFramesOnStack(). + std::unordered_map callsite_deopt_exits_; + + // Backpointer to the CompiledFunction that owns this CodeRuntime. + // Set after CompiledFunction::create() in makeCompiledFunction(). + BorrowedRef compiled_function_; + + GenResumeFunc gen_resume_entry_{nullptr}; + int frame_size_{-1}; + uint32_t spill_words_{0}; DebugInfo debug_info_; }; -} // namespace jit +} // namespace cinderx::jit diff --git a/cinderx/Jit/codegen/annotations.cpp b/cinderx/Jit/codegen/annotations.cpp index 6e38092b7..6e2cba092 100644 --- a/cinderx/Jit/codegen/annotations.cpp +++ b/cinderx/Jit/codegen/annotations.cpp @@ -9,7 +9,7 @@ #include #include -namespace jit::codegen { +namespace cinderx::jit::codegen { std::string Annotations::disassembleSection( void* entry, @@ -17,7 +17,7 @@ std::string Annotations::disassembleSection( CodeSection section) { JIT_CHECK( getConfig().log.dump_asm, - "Annotations are not recorded without -X jit-dump-asm"); + "Annotations are not recorded without -X cinderx-jit-dump-asm"); auto text = code.sectionByName(codeSectionName(section)); if (text == nullptr) { return ""; @@ -37,7 +37,11 @@ std::string Annotations::disassembleSection( } auto inserted = annot_bounds.emplace(begin, std::make_pair(&annot, end)).second; - JIT_DCHECK(inserted, "Duplicate start address for annotation"); + JIT_DCHECK( + inserted, + "Duplicate start address for annotation {} {}", + annot.str, + annot_bounds[begin].first->str); } Annotation* prev_annot = nullptr; @@ -46,6 +50,12 @@ std::string Annotations::disassembleSection( std::stringstream result; Disassembler dis(section_start, size); + if (dis.cursor() == nullptr) { + // Should already be handled by ENABLE_DISASSEMBLER ifdefs, but putting this + // defensive check here just in case. + return ""; + } + dis.setPrintInstBytes(false); for (auto cursor = section_start, end = cursor + size; cursor < end;) { auto new_annot = prev_annot; @@ -75,7 +85,7 @@ std::string Annotations::disassembleSection( auto prev_hir = prev_annot ? prev_annot->instr : nullptr; if (new_hir != nullptr && new_hir != prev_hir) { annot_str = - hir::HIRPrinter().setFullSnapshots(true).ToString(*new_hir); + hir::HIRPrinter().setFullSnapshots(true).toString(*new_hir); } else if (!new_annot->str.empty()) { annot_str = new_annot->str; } @@ -107,4 +117,4 @@ std::string Annotations::disassemble( return result; } -} // namespace jit::codegen +} // namespace cinderx::jit::codegen diff --git a/cinderx/Jit/codegen/annotations.h b/cinderx/Jit/codegen/annotations.h index 70d494d1e..6b0b25ea1 100644 --- a/cinderx/Jit/codegen/annotations.h +++ b/cinderx/Jit/codegen/annotations.h @@ -11,7 +11,7 @@ #include #include -namespace jit { +namespace cinderx::jit { namespace hir { class Instr; } @@ -82,4 +82,4 @@ class Annotations { }; } // namespace codegen -} // namespace jit +} // namespace cinderx::jit diff --git a/cinderx/Jit/codegen/arch.cpp b/cinderx/Jit/codegen/arch.cpp index 193faec1c..9a151950e 100644 --- a/cinderx/Jit/codegen/arch.cpp +++ b/cinderx/Jit/codegen/arch.cpp @@ -6,11 +6,11 @@ using namespace asmjit; -namespace jit::codegen::arch { +namespace cinderx::jit::codegen::arch { // Attempt to build a pointer using an offset from a base register. If it is // not possible to do so, return std::nullopt. -static std::optional +std::optional ptr_offset_try(const a64::Gp& base, int32_t offset, AccessSize access_size) { if (offset >= -256 && offset < 256) { // Unscaled immediate offset @@ -69,14 +69,77 @@ a64::Mem ptr_resolve( return a64::ptr(base, scratch); } -} // namespace jit::codegen::arch +void cmp_immediate(a64::Builder* as, const arch::Gp& reg, uint64_t imm) { + if (arm::Utils::isAddSubImm(imm)) { + as->cmp(reg, imm); + } else if (arm::Utils::isAddSubImm(-imm)) { + as->cmn(reg, -imm); + } else { + arch::Gp scratch = arch::reg_scratch_0; + if (reg.isGpW()) { + scratch = scratch.w(); + } + + as->mov(scratch, imm); + as->cmp(reg, scratch); + } +} + +void add_immediate( + a64::Builder* as, + const a64::Gp& res, + const a64::Gp& lhs, + uint64_t rhsi) { + if (rhsi == 0) { + if (res != lhs) { + as->mov(res, lhs); + } + } else if (arm::Utils::isAddSubImm(rhsi)) { + as->add(res, lhs, rhsi); + } else { + as->mov(arch::reg_scratch_0, rhsi); + as->add(res, lhs, arch::reg_scratch_0); + } +} + +void sub_immediate( + a64::Builder* as, + const a64::Gp& res, + const a64::Gp& lhs, + uint64_t rhsi) { + if (rhsi == 0) { + if (res != lhs) { + as->mov(res, lhs); + } + } else if (arm::Utils::isAddSubImm(rhsi)) { + as->sub(res, lhs, rhsi); + } else { + as->mov(arch::reg_scratch_0, rhsi); + as->sub(res, lhs, arch::reg_scratch_0); + } +} + +void add_signed_immediate( + a64::Builder* as, + const a64::Gp& res, + const a64::Gp& lhs, + int64_t rhsi) { + uint64_t rshu = static_cast(rhsi); + if (rhsi >= 0) { + add_immediate(as, res, lhs, rshu); + } else { + sub_immediate(as, res, lhs, -rshu); + } +} + +} // namespace cinderx::jit::codegen::arch #endif -namespace jit::codegen { +namespace cinderx::jit::codegen { std::ostream& operator<<(std::ostream& out, const PhyLocation& loc) { return out << loc.toString(); } -} // namespace jit::codegen +} // namespace cinderx::jit::codegen diff --git a/cinderx/Jit/codegen/arch.h b/cinderx/Jit/codegen/arch.h index d1fd5c8c5..ecd0c7471 100644 --- a/cinderx/Jit/codegen/arch.h +++ b/cinderx/Jit/codegen/arch.h @@ -2,13 +2,13 @@ #pragma once -// NOLINTNEXTLINE(facebook-unused-include-check) -#include "cinderx/Jit/codegen/arch/detection.h" -#include "fmt/ostream.h" +#include "cinderx/Common/define.h" #include +#include #include +#include #if defined(CINDER_X86_64) @@ -18,7 +18,7 @@ #include #include -namespace jit::codegen::arch { +namespace cinderx::jit::codegen::arch { using Builder = asmjit::x86::Builder; using Emitter = asmjit::x86::Emitter; @@ -33,6 +33,7 @@ using EmitterExplicitT = asmjit::x86::EmitterExplicitT; // If you change this register you'll also need to change the deopt // trampoline code that saves all registers. constexpr auto reg_scratch_deopt = asmjit::x86::r15; +constexpr auto reg_scratch_deopt_loc = R15; constexpr auto reg_scratch_0_loc = RAX; @@ -43,7 +44,9 @@ constexpr auto reg_double_auxilary_return_loc = XMM1; constexpr auto reg_frame_pointer_loc = RBP; constexpr auto reg_stack_pointer_loc = RSP; -} // namespace jit::codegen::arch +constexpr auto fp = asmjit::x86::rbp; + +} // namespace cinderx::jit::codegen::arch #elif defined(CINDER_AARCH64) @@ -54,7 +57,7 @@ constexpr auto reg_stack_pointer_loc = RSP; #include #include -namespace jit::codegen::arch { +namespace cinderx::jit::codegen::arch { using Builder = asmjit::a64::Builder; using Emitter = asmjit::a64::Emitter; @@ -69,12 +72,21 @@ using EmitterExplicitT = asmjit::a64::EmitterExplicitT; // If you change this register you'll also need to change the deopt // trampoline code that saves all registers. constexpr auto reg_scratch_deopt = asmjit::a64::x28; +constexpr auto reg_scratch_deopt_loc = X28; -constexpr auto reg_scratch_0 = asmjit::a64::x12; -constexpr auto reg_scratch_1 = asmjit::a64::x13; +constexpr auto reg_scratch_0 = asmjit::a64::x13; +constexpr auto reg_scratch_1 = asmjit::a64::x14; constexpr auto reg_scratch_br = asmjit::a64::x16; +constexpr auto reg_scratch_br_loc = X16; + +constexpr auto reg_scratch_0_loc = X13; +constexpr auto reg_scratch_1_loc = X14; + +constexpr auto reg_fp_scratch_0 = asmjit::a64::d16; +constexpr auto reg_fp_scratch_1 = asmjit::a64::d17; -constexpr auto reg_scratch_0_loc = X12; +constexpr auto reg_fp_scratch_0_loc = D16; +constexpr auto reg_fp_scratch_1_loc = D17; constexpr auto reg_general_return_loc = X0; constexpr auto reg_general_auxilary_return_loc = X1; @@ -86,13 +98,16 @@ constexpr auto reg_stack_pointer_loc = SP; constexpr auto fp = asmjit::a64::x29; constexpr auto lr = asmjit::a64::x30; -} // namespace jit::codegen::arch +// Size of the AArch64 frame record: saved FP + LR (two 64-bit registers). +constexpr int kFrameRecordSize = 2 * sizeof(void*); + +} // namespace cinderx::jit::codegen::arch #else #include "cinderx/Jit/codegen/arch/unknown.h" -namespace jit::codegen::arch { +namespace cinderx::jit::codegen::arch { class Builder : public asmjit::BaseBuilder { public: @@ -123,16 +138,25 @@ constexpr auto reg_double_auxilary_return_loc = D1; constexpr auto reg_frame_pointer_loc = R3; constexpr auto reg_stack_pointer_loc = SP; -} // namespace jit::codegen::arch +} // namespace cinderx::jit::codegen::arch #endif #if defined(CINDER_AARCH64) -namespace jit::codegen::arch { +namespace cinderx::jit::codegen::arch { enum class AccessSize : int32_t { k8 = 1, k16 = 2, k32 = 4, k64 = 8 }; +// Sentinel for Environ::sp_to_fp_delta meaning SP is not at its frame +// position, so frame slots can only be reached through FP. +constexpr int32_t kSpPositionUnknown = -1; + +std::optional ptr_offset_try( + const asmjit::a64::Gp& base, + int32_t offset, + AccessSize access_size); + asmjit::a64::Mem ptr_offset( const asmjit::a64::Gp& base, int32_t offset, @@ -145,25 +169,53 @@ asmjit::a64::Mem ptr_resolve( const asmjit::a64::Gp& scratch, AccessSize access_size = AccessSize::k64); -} // namespace jit::codegen::arch +} // namespace cinderx::jit::codegen::arch + +namespace cinderx::jit::codegen::arch { + +void cmp_immediate( + asmjit::a64::Builder* as, + const asmjit::a64::Gp& reg, + uint64_t imm); + +void add_immediate( + asmjit::a64::Builder* as, + const asmjit::a64::Gp& res, + const asmjit::a64::Gp& lhs, + uint64_t rhsi); + +void sub_immediate( + asmjit::a64::Builder* as, + const asmjit::a64::Gp& res, + const asmjit::a64::Gp& lhs, + uint64_t rhsi); + +void add_signed_immediate( + asmjit::a64::Builder* as, + const asmjit::a64::Gp& res, + const asmjit::a64::Gp& lhs, + int64_t rhsi); + +} // namespace cinderx::jit::codegen::arch #endif -namespace jit::codegen { +namespace cinderx::jit::codegen { std::ostream& operator<<(std::ostream& out, const PhyLocation& loc); -} // namespace jit::codegen +} // namespace cinderx::jit::codegen -inline auto format_as(jit::codegen::RegId reg) { +inline auto format_as(cinderx::jit::codegen::RegId reg) { return fmt::underlying(reg); } namespace std { template <> -struct hash { - std::size_t operator()(jit::codegen::PhyLocation const& s) const noexcept { +struct hash { + std::size_t operator()( + cinderx::jit::codegen::PhyLocation const& s) const noexcept { return s.loc; } }; @@ -171,4 +223,5 @@ struct hash { } // namespace std template <> -struct fmt::formatter : fmt::ostream_formatter {}; +struct fmt::formatter + : fmt::ostream_formatter {}; diff --git a/cinderx/Jit/codegen/arch/aarch64.cpp b/cinderx/Jit/codegen/arch/aarch64.cpp index d009b64ba..c002757c6 100644 --- a/cinderx/Jit/codegen/arch/aarch64.cpp +++ b/cinderx/Jit/codegen/arch/aarch64.cpp @@ -2,11 +2,11 @@ #include "cinderx/Jit/codegen/arch/aarch64.h" -#include "cinderx/Jit/codegen/arch/detection.h" +#include "cinderx/Common/define.h" #ifdef CINDER_AARCH64 -namespace jit::codegen { +namespace cinderx::jit::codegen { PhyLocation PhyLocation::parse(std::string_view name) { #define FIND_GP_REG(V64, V32) \ @@ -33,7 +33,7 @@ PhyLocation PhyLocation::parse(std::string_view name) { } std::string PhyLocation::toString() const { - if (is_memory()) { + if (isMemory()) { return fmt::format("[X29({})]", loc); } else if (bitSize == 32 || bitSize == 16 || bitSize == 8) { return std::string{name32(static_cast(loc))}; @@ -41,6 +41,6 @@ std::string PhyLocation::toString() const { return std::string{name(static_cast(loc))}; } -} // namespace jit::codegen +} // namespace cinderx::jit::codegen #endif diff --git a/cinderx/Jit/codegen/arch/aarch64.h b/cinderx/Jit/codegen/arch/aarch64.h index 55d0fa91c..50183a64e 100644 --- a/cinderx/Jit/codegen/arch/aarch64.h +++ b/cinderx/Jit/codegen/arch/aarch64.h @@ -4,6 +4,8 @@ #include "cinderx/Common/log.h" #include "cinderx/Common/util.h" +#include "cinderx/Jit/codegen/arch/phy_location.h" +#include "cinderx/Jit/codegen/arch/register_set.h" #include @@ -11,7 +13,7 @@ #include #include -namespace jit::codegen { +namespace cinderx::jit::codegen { #define FOREACH_GP(X) \ X(X0, W0) \ @@ -129,9 +131,10 @@ constexpr std::string_view name32(RegId id) { } // A physical location (register or stack slot). If this represents a stack -// slot (is_memory() is true) then `loc` is relative to X29 (the frame pointer). -struct PhyLocation { - static constexpr int REG_INVALID = -1; +// slot (isMemory() is true) then `loc` is relative to X29 (the frame pointer). +struct PhyLocation : PhyLocationBase { + using Base = PhyLocationBase; + using Base::Base; #define DEFINE_REG(V, ...) static constexpr int V = raw(RegId::V); FOREACH_GP(DEFINE_REG) @@ -144,53 +147,7 @@ struct PhyLocation { // support parsing stack slots. static PhyLocation parse(std::string_view name); - int32_t loc{REG_INVALID}; - uint32_t bitSize{64}; - - PhyLocation() = default; - - /* implicit */ constexpr PhyLocation(RegId reg, size_t size = 64) - : PhyLocation{static_cast(reg), size} {} - - /* implicit */ constexpr PhyLocation(RegId reg, int size) - : PhyLocation{static_cast(reg), static_cast(size)} {} - - /* implicit */ constexpr PhyLocation(int loc, size_t size = 64) - : loc{loc}, bitSize{static_cast(size)} {} - - /* implicit */ constexpr PhyLocation(int loc, int size) - : PhyLocation{loc, static_cast(size)} {} - - bool is_memory() const { - return loc < 0; - } - - bool is_register() const { - return loc >= 0 && loc < NUM_REGS; - } - - bool is_gp_register() const { - return is_register() && loc < VECD_REG_BASE; - } - - bool is_fp_register() const { - return is_register() && loc >= VECD_REG_BASE && loc < NUM_REGS; - } - std::string toString() const; - - // Comparisons are based only on the register ID. - // - // TODO: This doesn't account for aliasing in stack slots, e.g. - // PhyLocation(loc=-8, bitSize=64) and PhyLocation(loc=-12, bitSize=32). - - bool operator==(const PhyLocation& rhs) const { - return loc == rhs.loc; - } - - bool operator!=(const PhyLocation& rhs) const { - return loc != rhs.loc; - } }; // Define global definitions like `X0` and `D0`. @@ -207,96 +164,7 @@ constexpr PhyLocation SP{RegId::SP, 64}; #undef DEFINE_PHY_GP_REG #undef DEFINE_PHY_VECD_REG -class PhyRegisterSet { - public: - constexpr PhyRegisterSet() : rs_(0ULL) {} - explicit constexpr PhyRegisterSet(PhyLocation r) : rs_(0ULL) { - rs_ |= (1ULL << r.loc); - } - - constexpr PhyRegisterSet operator|(PhyLocation reg) const { - PhyRegisterSet set; - set.rs_ = rs_ | (1ULL << reg.loc); - return set; - } - - constexpr PhyRegisterSet operator|(const PhyRegisterSet& rs) const { - PhyRegisterSet res; - res.rs_ = rs_ | rs.rs_; - return res; - } - - PhyRegisterSet& operator|=(const PhyRegisterSet& rs) { - rs_ |= rs.rs_; - return *this; - } - - constexpr PhyRegisterSet operator-(PhyLocation rs) const { - return operator-(PhyRegisterSet(rs)); - } - - constexpr PhyRegisterSet operator-(PhyRegisterSet rs) const { - PhyRegisterSet set; - set.rs_ = rs_ & ~(rs.rs_); - return set; - } - - constexpr PhyRegisterSet operator&(PhyRegisterSet rs) const { - PhyRegisterSet set; - set.rs_ = rs_ & rs.rs_; - return set; - } - - constexpr bool operator==(const PhyRegisterSet& rs) const { - return rs_ == rs.rs_; - } - - constexpr bool Empty() const { - return rs_ == 0ULL; - } - - int count() const { - return popcount(rs_); - } - - PhyLocation GetFirst() const { - JIT_DCHECK(rs_ != 0, "__builtin_ctzll(0) is undefined"); - return __builtin_ctzll(rs_); - } - - PhyLocation GetLast() const { - return GetLastBit(); - } - - constexpr void RemoveFirst() { - rs_ &= (rs_ - 1ULL); - } - - constexpr void RemoveLast() { - rs_ &= ~(1ULL << GetLastBit()); - } - - void Set(PhyLocation reg) { - rs_ |= (1ULL << reg.loc); - } - void Reset(PhyLocation reg) { - rs_ &= ~(1ULL << reg.loc); - } - void ResetAll() { - rs_ = 0ULL; - } - - bool Has(PhyLocation reg) const { - return rs_ & (1ULL << reg.loc); - } - - private: - uint64_t rs_; - - int GetLastBit() const { - return (sizeof(rs_) * CHAR_BIT - 1) - __builtin_clzll(rs_); - } -}; +using PhyRegisterSet = RegisterSet; #define ADD_REG(v, ...) | PhyLocation::v constexpr PhyRegisterSet ALL_GP_REGISTERS = @@ -307,8 +175,8 @@ constexpr PhyRegisterSet ALL_REGISTERS = ALL_GP_REGISTERS | ALL_VECD_REGISTERS; #undef ADD_REG constexpr PhyRegisterSet DISALLOWED_REGISTERS = PhyRegisterSet(X29) /* FP */ | - X30 /* LR */ | XZR /* zero */ | X12 /* scratch0 */ | X13 /* scratch1 */ | - X16 /* IP0 */; + X30 /* LR */ | XZR /* zero */ | X13 /* scratch0 */ | X14 /* scratch1 */ | + X16 /* IP0 */ | D16 /* fp_scratch0 */ | D17 /* fp_scratch1 */; constexpr PhyRegisterSet INIT_REGISTERS = ALL_REGISTERS - DISALLOWED_REGISTERS; @@ -330,7 +198,10 @@ constexpr auto FP_ARGUMENT_REGS = // there. constexpr PhyLocation INITIAL_EXTRA_ARGS_REG = X10; constexpr PhyLocation INITIAL_TSTATE_REG = X11; + // This is often provided by the first argument in the vector call protocol. constexpr PhyLocation INITIAL_FUNC_REG = ARGUMENT_REGS[0]; -} // namespace jit::codegen +constexpr int kShadowSpaceSize = 0; + +} // namespace cinderx::jit::codegen diff --git a/cinderx/Jit/codegen/arch/detection.h b/cinderx/Jit/codegen/arch/detection.h deleted file mode 100644 index b2e063870..000000000 --- a/cinderx/Jit/codegen/arch/detection.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. - -#pragma once - -#if defined(__x86_64__) - -#define CINDER_X86_64 - -#elif defined(__aarch64__) - -#define CINDER_AARCH64 - -// This is here until we have aarch64 support everywhere. -#define CINDER_UNSUPPORTED - -#else - -#define CINDER_UNKNOWN - -// This macro is a marker for places that need platform-specific code. -#define CINDER_UNSUPPORTED - -#endif diff --git a/cinderx/Jit/codegen/arch/phy_location.h b/cinderx/Jit/codegen/arch/phy_location.h new file mode 100644 index 000000000..755d78569 --- /dev/null +++ b/cinderx/Jit/codegen/arch/phy_location.h @@ -0,0 +1,56 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#pragma once + +#include +#include + +namespace cinderx::jit::codegen { + +template +struct PhyLocationBase { + static constexpr int REG_INVALID = -1; + + int32_t loc{REG_INVALID}; + uint32_t bitSize{64}; + + PhyLocationBase() = default; + + /* implicit */ constexpr PhyLocationBase(RegIdType reg, size_t size = 64) + : PhyLocationBase{static_cast(reg), size} {} + + /* implicit */ constexpr PhyLocationBase(RegIdType reg, int size) + : PhyLocationBase{static_cast(reg), static_cast(size)} {} + + /* implicit */ constexpr PhyLocationBase(int loc, size_t size = 64) + : loc{loc}, bitSize{static_cast(size)} {} + + /* implicit */ constexpr PhyLocationBase(int loc, int size) + : PhyLocationBase{loc, static_cast(size)} {} + + constexpr bool isMemory() const { + return loc < 0; + } + + constexpr bool isRegister() const { + return loc >= 0 && loc < NumRegs; + } + + constexpr bool isGpRegister() const { + return isRegister() && loc < VecDRegBase; + } + + constexpr bool isFpRegister() const { + return isRegister() && loc >= VecDRegBase; + } + + // Comparisons are based only on the register ID. + // + // TODO: This doesn't account for aliasing in stack slots, e.g. + // PhyLocation(loc=-8, bitSize=64) and PhyLocation(loc=-12, bitSize=32). + constexpr bool operator==(const PhyLocationBase& rhs) const { + return loc == rhs.loc; + } +}; + +} // namespace cinderx::jit::codegen diff --git a/cinderx/Jit/codegen/arch/register_set.h b/cinderx/Jit/codegen/arch/register_set.h new file mode 100644 index 000000000..6987090d4 --- /dev/null +++ b/cinderx/Jit/codegen/arch/register_set.h @@ -0,0 +1,112 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#pragma once + +#include +#include +#include +#include + +namespace cinderx::jit::codegen { + +template +class RegisterSet { + public: + constexpr RegisterSet() = default; + explicit constexpr RegisterSet(PhyLocationType reg) : rs_{bitAt(reg.loc)} {} + explicit constexpr RegisterSet(std::span regs) { + for (auto reg : regs) { + rs_ |= bitAt(reg.loc); + } + } + + constexpr RegisterSet operator|(PhyLocationType reg) const { + RegisterSet set; + set.rs_ = rs_ | bitAt(reg.loc); + return set; + } + + constexpr RegisterSet operator|(const RegisterSet& rs) const { + RegisterSet res; + res.rs_ = rs_ | rs.rs_; + return res; + } + + RegisterSet& operator|=(const RegisterSet& rs) { + rs_ |= rs.rs_; + return *this; + } + + constexpr RegisterSet operator-(PhyLocationType reg) const { + return operator-(RegisterSet(reg)); + } + + constexpr RegisterSet operator-(RegisterSet rs) const { + RegisterSet set; + set.rs_ = rs_ & ~(rs.rs_); + return set; + } + + constexpr RegisterSet operator&(RegisterSet rs) const { + RegisterSet set; + set.rs_ = rs_ & rs.rs_; + return set; + } + + constexpr bool operator==(const RegisterSet& rs) const { + return rs_ == rs.rs_; + } + + constexpr bool empty() const { + return rs_ == StorageType{0}; + } + + constexpr int count() const { + return std::popcount(rs_); + } + + constexpr PhyLocationType getFirst() const { + return std::countr_zero(rs_); + } + + constexpr PhyLocationType getLast() const { + return getLastBit(); + } + + constexpr void removeFirst() { + rs_ &= (rs_ - StorageType{1}); + } + + constexpr void removeLast() { + rs_ &= ~bitAt(getLastBit()); + } + + constexpr void set(PhyLocationType reg) { + rs_ |= bitAt(reg.loc); + } + + constexpr void reset(PhyLocationType reg) { + rs_ &= ~bitAt(reg.loc); + } + + constexpr void resetAll() { + rs_ = StorageType{0}; + } + + constexpr bool has(PhyLocationType reg) const { + return rs_ & bitAt(reg.loc); + } + + private: + StorageType rs_{0}; + + static constexpr StorageType bitAt(int n) { + return StorageType{1} << n; + } + + constexpr int getLastBit() const { + return (sizeof(rs_) * CHAR_BIT - 1) - std::countl_zero(rs_); + } +}; + +} // namespace cinderx::jit::codegen diff --git a/cinderx/Jit/codegen/arch/unknown.cpp b/cinderx/Jit/codegen/arch/unknown.cpp index f702090d4..bff090955 100644 --- a/cinderx/Jit/codegen/arch/unknown.cpp +++ b/cinderx/Jit/codegen/arch/unknown.cpp @@ -2,12 +2,11 @@ #include "cinderx/Jit/codegen/arch/unknown.h" -// NOLINTNEXTLINE(facebook-unused-include-check) -#include "cinderx/Jit/codegen/arch/detection.h" +#include "cinderx/Common/define.h" #ifdef CINDER_UNKNOWN -namespace jit::codegen { +namespace cinderx::jit::codegen { PhyLocation PhyLocation::parse(std::string_view name) { #define FIND_GP_REG(V) \ @@ -31,12 +30,12 @@ PhyLocation PhyLocation::parse(std::string_view name) { } std::string PhyLocation::toString() const { - if (is_memory()) { + if (isMemory()) { return fmt::format("[FP({})]", loc); } return std::string{name(static_cast(loc))}; } -} // namespace jit::codegen +} // namespace cinderx::jit::codegen #endif diff --git a/cinderx/Jit/codegen/arch/unknown.h b/cinderx/Jit/codegen/arch/unknown.h index 240b7b578..7db6139a8 100644 --- a/cinderx/Jit/codegen/arch/unknown.h +++ b/cinderx/Jit/codegen/arch/unknown.h @@ -4,15 +4,16 @@ #include "cinderx/Common/log.h" #include "cinderx/Common/util.h" +#include "cinderx/Jit/codegen/arch/phy_location.h" +#include "cinderx/Jit/codegen/arch/register_set.h" #include #include -#include #include #include -namespace jit::codegen { +namespace cinderx::jit::codegen { #define FOREACH_GP(X) \ X(R0) \ @@ -61,9 +62,10 @@ constexpr std::string_view name(RegId id) { } // A physical location (register or stack slot). If this represents a stack -// slot (is_memory() is true) then `loc` is relative to R3. -struct PhyLocation { - static constexpr int REG_INVALID = -1; +// slot (isMemory() is true) then `loc` is relative to R3. +struct PhyLocation : PhyLocationBase { + using Base = PhyLocationBase; + using Base::Base; #define DEFINE_REG(V) static constexpr int V = raw(RegId::V); FOREACH_GP(DEFINE_REG) @@ -76,48 +78,7 @@ struct PhyLocation { // support parsing stack slots. static PhyLocation parse(std::string_view name); - int32_t loc{REG_INVALID}; - uint32_t bitSize{64}; - - PhyLocation() = default; - - /* implicit */ constexpr PhyLocation(RegId reg, size_t size = 64) - : PhyLocation{static_cast(reg), size} {} - - /* implicit */ constexpr PhyLocation(RegId reg, int size) - : PhyLocation{static_cast(reg), static_cast(size)} {} - - /* implicit */ constexpr PhyLocation(int loc, size_t size = 64) - : loc{loc}, bitSize{static_cast(size)} {} - - /* implicit */ constexpr PhyLocation(int loc, int size) - : PhyLocation{loc, static_cast(size)} {} - - bool is_memory() const { - return loc < 0; - } - - bool is_register() const { - return loc >= 0; - } - - bool is_gp_register() const { - return is_register() && loc < VECD_REG_BASE; - } - - bool is_fp_register() const { - return is_register() && loc >= VECD_REG_BASE; - } - std::string toString() const; - - bool operator==(const PhyLocation& rhs) const { - return loc == rhs.loc; - } - - bool operator!=(const PhyLocation& rhs) const { - return loc != rhs.loc; - } }; // Define global definitions like `R0` and `D0`. @@ -127,100 +88,9 @@ FOREACH_GP(DEFINE_PHY_REG) FOREACH_VECD(DEFINE_PHY_REG) constexpr PhyLocation SP{RegId::SP, 64}; -#undef DEFINE_PHY_GP_REG -#undef DEFINE_PHY_VECD_REG - -class PhyRegisterSet { - public: - constexpr PhyRegisterSet() = default; - explicit constexpr PhyRegisterSet(PhyLocation r) : rs_(0) { - rs_ |= (1 << r.loc); - } - - constexpr PhyRegisterSet operator|(PhyLocation reg) const { - PhyRegisterSet set; - set.rs_ = rs_ | (1 << reg.loc); - return set; - } - - constexpr PhyRegisterSet operator|(const PhyRegisterSet& rs) const { - PhyRegisterSet res; - res.rs_ = rs_ | rs.rs_; - return res; - } - - PhyRegisterSet& operator|=(const PhyRegisterSet& rs) { - rs_ |= rs.rs_; - return *this; - } +#undef DEFINE_PHY_REG - constexpr PhyRegisterSet operator-(PhyLocation rs) const { - return operator-(PhyRegisterSet(rs)); - } - - constexpr PhyRegisterSet operator-(PhyRegisterSet rs) const { - PhyRegisterSet set; - set.rs_ = rs_ & ~(rs.rs_); - return set; - } - - constexpr PhyRegisterSet operator&(PhyRegisterSet rs) const { - PhyRegisterSet set; - set.rs_ = rs_ & rs.rs_; - return set; - } - - constexpr bool operator==(const PhyRegisterSet& rs) const { - return rs_ == rs.rs_; - } - - constexpr bool Empty() const { - return rs_ == 0ULL; - } - - constexpr int count() const { - return std::popcount(rs_); - } - - constexpr PhyLocation GetFirst() const { - return std::countr_zero(rs_); - } - - constexpr PhyLocation GetLast() const { - return GetLastBit(); - } - - constexpr void RemoveFirst() { - rs_ &= (rs_ - 1); - } - - constexpr void RemoveLast() { - rs_ &= ~(1U << GetLastBit()); - } - - constexpr void Set(PhyLocation reg) { - rs_ |= (1 << reg.loc); - } - - constexpr void Reset(PhyLocation reg) { - rs_ &= ~(1 << reg.loc); - } - - constexpr void ResetAll() { - rs_ = 0; - } - - constexpr bool Has(PhyLocation reg) const { - return rs_ & (1 << reg.loc); - } - - private: - uint32_t rs_{0}; - - constexpr int GetLastBit() const { - return (sizeof(rs_) * CHAR_BIT - 1) - std::countl_zero(rs_); - } -}; +using PhyRegisterSet = RegisterSet; #define ADD_REG(V) | PhyLocation::V constexpr PhyRegisterSet ALL_GP_REGISTERS = @@ -247,4 +117,4 @@ constexpr PhyLocation INITIAL_TSTATE_REG = R2; // This is often provided by the first argument in the vector call protocol. constexpr PhyLocation INITIAL_FUNC_REG = ARGUMENT_REGS[0]; -} // namespace jit::codegen +} // namespace cinderx::jit::codegen diff --git a/cinderx/Jit/codegen/arch/x86_64.cpp b/cinderx/Jit/codegen/arch/x86_64.cpp index 4dddcadd0..d388c88e2 100644 --- a/cinderx/Jit/codegen/arch/x86_64.cpp +++ b/cinderx/Jit/codegen/arch/x86_64.cpp @@ -2,12 +2,11 @@ #include "cinderx/Jit/codegen/arch/x86_64.h" -// NOLINTNEXTLINE(facebook-unused-include-check) -#include "cinderx/Jit/codegen/arch/detection.h" +#include "cinderx/Common/define.h" #ifdef CINDER_X86_64 -namespace jit::codegen { +namespace cinderx::jit::codegen { PhyLocation PhyLocation::parse(std::string_view name) { #define FIND_GP_REG(V64, V32, V16, V8) \ @@ -37,7 +36,7 @@ PhyLocation PhyLocation::parse(std::string_view name) { } std::string PhyLocation::toString() const { - if (is_memory()) { + if (isMemory()) { return fmt::format("[RBP({})]", loc); } else if (bitSize == 32) { return std::string{name32(static_cast(loc))}; @@ -49,6 +48,6 @@ std::string PhyLocation::toString() const { return std::string{name(static_cast(loc))}; } -} // namespace jit::codegen +} // namespace cinderx::jit::codegen #endif diff --git a/cinderx/Jit/codegen/arch/x86_64.h b/cinderx/Jit/codegen/arch/x86_64.h index 238a34d65..a84815aa1 100644 --- a/cinderx/Jit/codegen/arch/x86_64.h +++ b/cinderx/Jit/codegen/arch/x86_64.h @@ -4,15 +4,16 @@ #include "cinderx/Common/log.h" #include "cinderx/Common/util.h" +#include "cinderx/Jit/codegen/arch/phy_location.h" +#include "cinderx/Jit/codegen/arch/register_set.h" #include #include -#include #include #include -namespace jit::codegen { +namespace cinderx::jit::codegen { #define FOREACH_GP(X) \ X(RAX, EAX, AX, AL) \ @@ -121,9 +122,10 @@ constexpr std::string_view name8(RegId id) { } // A physical location (register or stack slot). If this represents a stack -// slot (is_memory() is true) then `loc` is relative to RBP. -struct PhyLocation { - static constexpr int REG_INVALID = -1; +// slot (isMemory() is true) then `loc` is relative to RBP. +struct PhyLocation : PhyLocationBase { + using Base = PhyLocationBase; + using Base::Base; #define DEFINE_REG(V, ...) static constexpr int V = raw(RegId::V); FOREACH_GP(DEFINE_REG) @@ -135,53 +137,7 @@ struct PhyLocation { // support parsing stack slots. static PhyLocation parse(std::string_view name); - int32_t loc{REG_INVALID}; - uint32_t bitSize{64}; - - PhyLocation() = default; - - /* implicit */ constexpr PhyLocation(RegId reg, size_t size = 64) - : PhyLocation{static_cast(reg), size} {} - - /* implicit */ constexpr PhyLocation(RegId reg, int size) - : PhyLocation{static_cast(reg), static_cast(size)} {} - - /* implicit */ constexpr PhyLocation(int loc, size_t size = 64) - : loc{loc}, bitSize{static_cast(size)} {} - - /* implicit */ constexpr PhyLocation(int loc, int size) - : PhyLocation{loc, static_cast(size)} {} - - bool is_memory() const { - return loc < 0; - } - - bool is_register() const { - return loc >= 0; - } - - bool is_gp_register() const { - return is_register() && loc < VECD_REG_BASE; - } - - bool is_fp_register() const { - return is_register() && loc >= VECD_REG_BASE; - } - std::string toString() const; - - // Comparisons are based only on the register ID. - // - // TODO: This doesn't account for aliasing in stack slots, e.g. - // PhyLocation(loc=-8, bitSize=64) and PhyLocation(loc=-12, bitSize=32). - - bool operator==(const PhyLocation& rhs) const { - return loc == rhs.loc; - } - - bool operator!=(const PhyLocation& rhs) const { - return loc != rhs.loc; - } }; // Define global definitions like `RAX` and `XMM0`. @@ -199,97 +155,7 @@ FOREACH_VECD(DEFINE_PHY_VECD_REG) #undef DEFINE_PHY_GP_REG #undef DEFINE_PHY_VECD_REG -class PhyRegisterSet { - public: - constexpr PhyRegisterSet() = default; - explicit constexpr PhyRegisterSet(PhyLocation r) { - rs_ |= (1 << r.loc); - } - - constexpr PhyRegisterSet operator|(PhyLocation reg) const { - PhyRegisterSet set; - set.rs_ = rs_ | (1 << reg.loc); - return set; - } - - constexpr PhyRegisterSet operator|(const PhyRegisterSet& rs) const { - PhyRegisterSet res; - res.rs_ = rs_ | rs.rs_; - return res; - } - - PhyRegisterSet& operator|=(const PhyRegisterSet& rs) { - rs_ |= rs.rs_; - return *this; - } - - constexpr PhyRegisterSet operator-(PhyLocation rs) const { - return operator-(PhyRegisterSet(rs)); - } - - constexpr PhyRegisterSet operator-(PhyRegisterSet rs) const { - PhyRegisterSet set; - set.rs_ = rs_ & ~(rs.rs_); - return set; - } - - constexpr PhyRegisterSet operator&(PhyRegisterSet rs) const { - PhyRegisterSet set; - set.rs_ = rs_ & rs.rs_; - return set; - } - - constexpr bool operator==(const PhyRegisterSet& rs) const { - return rs_ == rs.rs_; - } - - constexpr bool Empty() const { - return rs_ == 0; - } - - constexpr int count() const { - return std::popcount(rs_); - } - - constexpr PhyLocation GetFirst() const { - return std::countr_zero(rs_); - } - - constexpr PhyLocation GetLast() const { - return GetLastBit(); - } - - constexpr void RemoveFirst() { - rs_ &= (rs_ - 1); - } - - constexpr void RemoveLast() { - rs_ &= ~(1U << GetLastBit()); - } - - constexpr void Set(PhyLocation reg) { - rs_ |= (1 << reg.loc); - } - - constexpr void Reset(PhyLocation reg) { - rs_ &= ~(1 << reg.loc); - } - - constexpr void ResetAll() { - rs_ = 0; - } - - constexpr bool Has(PhyLocation reg) const { - return rs_ & (1 << reg.loc); - } - - private: - unsigned rs_{0}; - - constexpr int GetLastBit() const { - return (sizeof(rs_) * CHAR_BIT - 1) - std::countl_zero(rs_); - } -}; +using PhyRegisterSet = RegisterSet; #define ADD_REG(v, ...) | PhyLocation::v constexpr PhyRegisterSet ALL_GP_REGISTERS = @@ -303,17 +169,35 @@ constexpr PhyRegisterSet DISALLOWED_REGISTERS = PhyRegisterSet(RSP) | RBP; constexpr PhyRegisterSet INIT_REGISTERS = ALL_REGISTERS - DISALLOWED_REGISTERS; +#ifdef _WIN32 +// Windows x64: RDI and RSI are callee-saved; XMM6-XMM15 are callee-saved. constexpr PhyRegisterSet CALLER_SAVE_REGS = PhyRegisterSet(RAX) | RCX | RDX | - RSI | RDI | R8 | R9 | R10 | R11 | ALL_VECD_REGISTERS; + R8 | R9 | R10 | R11 | PhyRegisterSet(XMM0) | XMM1 | XMM2 | XMM3 | XMM4 | + XMM5; +#else +constexpr auto CALLER_SAVE_GP_REGS = + std::to_array({RAX, RCX, RDX, RSI, RDI, R8, R9, R10, R11}); + +constexpr PhyRegisterSet CALLER_SAVE_REGS = + PhyRegisterSet(CALLER_SAVE_GP_REGS) | ALL_VECD_REGISTERS; +#endif constexpr PhyRegisterSet CALLEE_SAVE_REGS = INIT_REGISTERS - CALLER_SAVE_REGS; +#ifdef _WIN32 +constexpr auto ARGUMENT_REGS = std::to_array({RCX, RDX, R8, R9}); +#else constexpr auto ARGUMENT_REGS = std::to_array({RDI, RSI, RDX, RCX, R8, R9}); +#endif constexpr auto RETURN_REGS = std::to_array({RAX, RDX}); +#ifdef _WIN32 +constexpr auto FP_ARGUMENT_REGS = std::to_array({XMM0, XMM1, XMM2, XMM3}); +#else constexpr auto FP_ARGUMENT_REGS = std::to_array({XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, XMM7}); +#endif // This is where the function prologue will initially store this data at entry // to the function body. The register allocator may move things around from @@ -323,4 +207,13 @@ constexpr PhyLocation INITIAL_TSTATE_REG = R11; // This is often provided by the first argument in the vector call protocol. constexpr PhyLocation INITIAL_FUNC_REG = ARGUMENT_REGS[0]; -} // namespace jit::codegen +// Shadow space (home space) required by the Windows x64 calling convention. +// The caller must always reserve 32 bytes above the return address for the +// callee to spill register arguments. System V does not use shadow space. +#ifdef _WIN32 +constexpr int kShadowSpaceSize = 32; +#else +constexpr int kShadowSpaceSize = 0; +#endif + +} // namespace cinderx::jit::codegen diff --git a/cinderx/Jit/codegen/autogen.cpp b/cinderx/Jit/codegen/autogen.cpp index 2d1070435..f0ab783ce 100644 --- a/cinderx/Jit/codegen/autogen.cpp +++ b/cinderx/Jit/codegen/autogen.cpp @@ -2,180 +2,321 @@ #include "cinderx/Jit/codegen/autogen.h" +#include "internal/pycore_pystate.h" + #include "cinderx/Common/util.h" #include "cinderx/Jit/code_patcher.h" #include "cinderx/Jit/codegen/arch.h" #include "cinderx/Jit/codegen/gen_asm_utils.h" +#include "cinderx/Jit/codegen/tsan.h" #include "cinderx/Jit/frame.h" #include "cinderx/Jit/generators_rt.h" +#include "cinderx/Jit/hir/hir.h" #include "cinderx/Jit/jit_rt.h" #include "cinderx/Jit/lir/instruction.h" #include "cinderx/Jit/lir/printer.h" - -#include -#include +#include "cinderx/module_state.h" using namespace asmjit; -using namespace jit::lir; -using namespace jit::codegen; +using namespace cinderx::jit::lir; +using namespace cinderx::jit::codegen; -namespace jit::codegen::autogen { - -#define ANY "*" +namespace cinderx::jit::codegen::autogen { namespace { -// Add a pattern to an existing trie tree. If the trie tree is nullptr, create a -// new one. -std::unique_ptr addPattern( - std::unique_ptr patterns, - const std::string& s, - PatternNode::func_t func) { - JIT_DCHECK(!s.empty(), "pattern string should not be empty."); - if (patterns == nullptr) { - patterns = std::make_unique(); - } +#if defined(CINDER_X86_64) +using AsmCondCode = x86::CondCode; +#elif defined(CINDER_AARCH64) +using AsmCondCode = arm::CondCode; +#endif - PatternNode* cur = patterns.get(); - for (auto& c : s) { - auto iter = cur->next.find(c); - if (iter == cur->next.end()) { - cur = cur->next.emplace(c, std::make_unique()) - .first->second.get(); - continue; - } - cur = iter->second.get(); +#if defined(CINDER_X86_64) || defined(CINDER_AARCH64) +// LIR spells its conditions the same way asmjit does, on both targets. +AsmCondCode asmCondCode(lir::Condition cond) { + switch (cond) { +#define TO_ASMJIT(NAME, ...) \ + case lir::Condition::k##NAME: \ + return AsmCondCode::k##NAME; + FOREACH_LIR_CONDITION(TO_ASMJIT) +#undef TO_ASMJIT + case lir::Condition::kInvalid: + break; } + JIT_THROW("Cannot encode invalid condition code {}", static_cast(cond)); +} - JIT_DCHECK(cur->func == nullptr, "Found duplicated pattern."); - cur->func = func; - - return patterns; +void emitBranchCC( + arch::Builder* as, + lir::Condition cond, + const asmjit::Label& label) { +#if defined(CINDER_X86_64) + as->j(asmCondCode(cond), label); +#else + as->b(asmCondCode(cond), label); +#endif } +#endif -// Find the function associated to the pattern given in s. -PatternNode::func_t findByPattern( - const PatternNode* patterns, - const std::string& s) { - auto cur = patterns; - if (s.empty()) { - // handle the special case of matching '*' with an empty string - auto iter = cur->next.find('*'); - if (iter != cur->next.end()) { - cur = iter->second.get(); - return cur->func; - } - } - for (auto& c : s) { - auto iter = cur->next.find(c); - if (iter != cur->next.end()) { - cur = iter->second.get(); - continue; - } +bool isMemoryMoveOperand(const lir::Operand* operand) { + return operand->isStack() || operand->isMem() || operand->isInd(); +} - iter = cur->next.find('?'); - if (iter != cur->next.end()) { - cur = iter->second.get(); - continue; - } +void checkMoveRelaxedOperandShape(const Instruction* instr) { + JIT_DCHECK( + instr->isMoveRelaxed(), "Expected kMoveRelaxed, got {}", instr->opname()); - iter = cur->next.find('*'); - if (iter != cur->next.end()) { - cur = iter->second.get(); - break; - } + auto* output = instr->output(); + auto* input = instr->getInput(0); - return nullptr; - } + bool is_valid_load = output->isReg() && isMemoryMoveOperand(input); + bool is_valid_store = + isMemoryMoveOperand(output) && (input->isReg() || input->isImm()); - return cur->func; + JIT_CHECK( + is_valid_load || is_valid_store, + "kMoveRelaxed only supports memory->register loads and " + "register/immediate->memory stores, got {} <- {}", + output->type(), + input->type()); } } // namespace -// this function generates operand patterns from the inputs and outputs -// of a given instruction instr and calls the correspoinding code generation -// functions. -void AutoTranslator::translateInstr(Environ* env, const Instruction* instr) - const { - auto opcode = instr->opcode(); - if (opcode == Instruction::kBind) { - return; - } - auto& instr_map = map_get(instr_rule_map_, opcode); +arch::Mem AsmIndirectOperandBuilder(const lir::Operand* operand) { + JIT_DCHECK(operand->isInd(), "operand should be an indirect reference"); - std::string pattern; - pattern.reserve(instr->getNumInputs() + instr->getNumOutputs()); +#if defined(CINDER_X86_64) + auto indirect = operand->getMemoryIndirect(); - if (instr->getNumOutputs()) { - auto operand = instr->output(); + lir::Operand* base = indirect->getBaseRegOperand(); + lir::Operand* index = indirect->getIndexRegOperand(); - switch (operand->type()) { - case OperandBase::kReg: - pattern += (operand->isVecD() ? "X" : "R"); - break; - case OperandBase::kStack: - case OperandBase::kMem: - case OperandBase::kInd: - pattern += "M"; - break; - default: - JIT_ABORT("Output operand has to be of type register or memory"); - } + if (index == nullptr) { + return asmjit::x86::ptr( + x86::gpq(base->getPhyRegister().loc), indirect->getOffset()); + } else { + return asmjit::x86::ptr( + x86::gpq(base->getPhyRegister().loc), + x86::gpq(index->getPhyRegister().loc), + indirect->getMultiplier(), + indirect->getOffset()); } +#elif defined(CINDER_AARCH64) + JIT_ABORT("Unreachable."); +#else + CINDER_UNSUPPORTED + return arch::Mem(); +#endif +} - instr->foreachInputOperand([&](const OperandBase* operand) { - switch (operand->type()) { - case OperandBase::kReg: - pattern += (operand->isVecD() ? "x" : "r"); - break; - case OperandBase::kStack: - case OperandBase::kMem: - case OperandBase::kInd: - pattern += "m"; - break; - case OperandBase::kImm: - pattern += "i"; - break; - case OperandBase::kLabel: - pattern += "b"; - break; - default: - JIT_ABORT( - "Illegal input type {} for instruction {}", - operand->type(), - *instr); +// Resolves the operand size in bits, respecting the instruction's +// OperandSizeType property. +int getOperandSize(const Instruction* instr, const lir::Operand* operand) { + switch (operandSizeType(instr->opcode())) { + case OperandSizeType::kAlways64: + return 64; + case OperandSizeType::kOut: { + // Match LIROperandMapper<0> behavior: use the output if present, + // otherwise use input 0 (which post-alloc rewrites may have resized). + if (instr->getNumOutputs() > 0) { + return static_cast(instr->output()->sizeInBits()); + } + return static_cast(instr->getInput(0)->sizeInBits()); } - }); + case OperandSizeType::kDefault: + default: + return static_cast(operand->sizeInBits()); + } +} - auto func = findByPattern(instr_map.get(), pattern); +int getOperandSizeInBytes( + const Instruction* instr, + const lir::Operand* operand) { + return getOperandSize(instr, operand) / 8; +} + +// Returns the appropriately-sized Gp register for a given operand, respecting +// the instruction's OperandSizeType property. +arch::Gp getReg(const Instruction* instr, const lir::Operand* operand) { JIT_CHECK( - func != nullptr, - "No pattern found for opcode {}: {}", - InstrProperty::getProperties(instr).name, - pattern); - func(env, instr); + operand->isReg(), + "Expected a register for getReg '{}' in '{}'", + *operand, + *instr); + int size = getOperandSize(instr, operand); + auto reg = operand->getPhyRegister().loc; +#if defined(CINDER_X86_64) + switch (size) { + case 8: + return asmjit::x86::gpb(reg); + case 16: + return asmjit::x86::gpw(reg); + case 32: + return asmjit::x86::gpd(reg); + case 64: + return asmjit::x86::gpq(reg); + } +#elif defined(CINDER_AARCH64) + switch (size) { + case 8: + case 16: + JIT_ABORT("Currently unsupported size."); + case 32: + return asmjit::a64::w(reg); + case 64: + return asmjit::a64::x(reg); + } +#else + CINDER_UNSUPPORTED +#endif + JIT_ABORT("Unexpected operand size {}", size); +} + +// Returns an arch::Mem for a given memory operand (stack, mem, or indirect), +// with size set according to the instruction's OperandSizeType property. +arch::Mem getMem(const Instruction* instr, const lir::Operand* operand) { +#if defined(CINDER_X86_64) + int size = getOperandSizeInBytes(instr, operand); + asmjit::x86::Mem memptr; + if (operand->isStack()) { + memptr = asmjit::x86::ptr(asmjit::x86::rbp, operand->getStackSlot().loc); + } else if (operand->isMem()) { + memptr = asmjit::x86::ptr( + reinterpret_cast(operand->getMemoryAddress())); + } else if (operand->isInd()) { + memptr = AsmIndirectOperandBuilder(operand); + } else { + JIT_ABORT("Unsupported operand type for getMem."); + } + memptr.setSize(size); + return memptr; +#elif defined(CINDER_AARCH64) + if (!operand->isStack()) { + JIT_ABORT("Unreachable."); + } + int32_t loc = operand->getStackSlot().loc; + JIT_CHECK(loc >= -256 && loc < 256, "Stack slot out of range"); + return arch::ptr_offset(arch::fp, loc); +#else + CINDER_UNSUPPORTED + return arch::Mem(); +#endif +} + +asmjit::Imm getImm(const lir::Operand* operand) { + return asmjit::Imm(operand->getConstant()); +} + +asmjit::Label getLabel(Environ* env, const lir::Operand* operand) { + if (operand->getDefine()->hasAsmLabel()) { + return operand->getDefine()->getAsmLabel(); + } + return map_get(env->block_label_map, operand->getBasicBlock()); } namespace { +#if defined(CINDER_AARCH64) + +// Address the frame slot at |loc| (a negative offset from FP), preferring +// SP-relative addressing so that the access usually needs no scratch base. +asmjit::a64::Mem getStackSlotPtr( + Environ* env, + int32_t loc, + const asmjit::a64::Gp& scratch = arch::reg_scratch_0, + arch::AccessSize access_size = arch::AccessSize::k64) { + if (env->sp_to_fp_delta != arch::kSpPositionUnknown) { + JIT_DCHECK( + loc < 0, "Frame slot offsets must be negative FP offsets, got {}", loc); + JIT_DCHECK( + loc + env->sp_to_fp_delta >= 0, + "SP-relative frame slot at {} must not be below SP (delta {})", + loc, + env->sp_to_fp_delta); + auto opt = + arch::ptr_offset_try(a64::sp, loc + env->sp_to_fp_delta, access_size); + if (opt.has_value()) { + return opt.value(); + } + } + return ptr_resolve(env->as, arch::fp, loc, scratch, access_size); +} + +#endif + void fillLiveValueLocations( CodeRuntime* code_runtime, std::size_t deopt_idx, const Instruction* instr, size_t begin_input, size_t end_input) { - ThreadedCompileSerialize guard; - DeoptMetadata& deopt_meta = code_runtime->getDeoptMetadata(deopt_idx); for (size_t i = begin_input; i < end_input; i++) { auto loc = instr->getInput(i)->getPhyRegOrStackSlot(); + JIT_THROW_IF( + loc.isFpRegister(), + "Deopt live value {} of {} is in vector register {}, which the deopt " + "trampoline does not spill", + i - begin_input, + instr->opname(), + loc.toString()); deopt_meta.live_values[i - begin_input].location = loc; } } +void fillCallSiteLiveValueLocations(Environ* env, const Instruction* instr) { + if constexpr (!kFreeThreadedBuild) { + return; + } + auto it = env->callsite_live_value_metadata.find(instr); + if (it == env->callsite_live_value_metadata.end()) { + const hir::Instr* hir_instr = instr->origin(); + // Assume if there is no HIR instruction then this site does not allow + // arbitrary execution. + if (hir_instr != nullptr && hir_instr->asDeoptBase() == nullptr) { + JIT_CHECK( + hir_instr->asCallSiteLiveValuesBase() == nullptr, + "Missing callsite live-value metadata for '{}'", + *hir_instr); + } + return; + } + + const Environ::CallSiteLiveValueMetadata& metadata = it->second; + JIT_CHECK( + metadata.live_values_instr != nullptr, + "Missing callsite live-value instruction"); + DeoptMetadata& deopt_meta = + env->code_rt->getDeoptMetadata(metadata.deopt_meta_index); + JIT_CHECK( + deopt_meta.live_values.size() == + metadata.live_values_instr->getNumInputs(), + "Callsite live-value count mismatch"); + fillLiveValueLocations( + env->code_rt, + metadata.deopt_meta_index, + metadata.live_values_instr, + 0, + metadata.live_values_instr->getNumInputs()); +} + +} // namespace + +#if defined(CINDER_AARCH64) +void translateA64GuardCC(Environ* env, const Instruction* instr) { + auto index = static_cast(instr->getInput(1)->getConstant()); + auto* block = map_get(env->deopt_exit_blocks, index); + auto label = map_get(env->block_label_map, block); + auto cond = static_cast(instr->getInput(0)->getConstant()); + + emitBranchCC(env->as, cond, label); + fillLiveValueLocations(env->code_rt, index, instr, 2, instr->getNumInputs()); +} +#endif + // Translate GUARD instruction -void TranslateGuard(Environ* env, const Instruction* instr) { +void translateGuard(Environ* env, const Instruction* instr) { #if defined(CINDER_X86_64) auto as = env->as; @@ -185,13 +326,15 @@ void TranslateGuard(Environ* env, const Instruction* instr) { // * guard var (physical register) (0 for AlwaysFail) // * target (for GuardIs and GuardType, and 0 for all others) - auto deopt_label = as->newLabel(); + auto index = static_cast(instr->getInput(1)->getConstant()); + auto* deopt_block = map_get(env->deopt_exit_blocks, index); + auto deopt_label = map_get(env->block_label_map, deopt_block); auto kind = instr->getInput(0)->getConstant(); arch::Gp reg = x86::rax; bool is_double = false; if (kind != kAlwaysFail) { - if (instr->getInput(2)->dataType() == jit::lir::OperandBase::kDouble) { + if (instr->getInput(2)->dataType() == jit::lir::Operand::kDouble) { JIT_CHECK(kind == kNotZero, "Only NotZero is supported for double"); auto vecd_reg = AutoTranslator::getVecD(instr->getInput(2)); as->ptest(vecd_reg, vecd_reg); @@ -242,11 +385,10 @@ void TranslateGuard(Environ* env, const Instruction* instr) { emit_cmp(reg); as->jne(deopt_label); break; - case kHasType: { + case kHasType: emit_cmp(x86::qword_ptr(reg, offsetof(PyObject, ob_type))); as->jne(deopt_label); break; - } } } #elif defined(CINDER_AARCH64) @@ -258,36 +400,29 @@ void TranslateGuard(Environ* env, const Instruction* instr) { // * guard var (physical register) (0 for AlwaysFail) // * target (for GuardIs and GuardType, and 0 for all others) - auto deopt_label = as->newLabel(); + auto index = static_cast(instr->getInput(1)->getConstant()); + auto* deopt_block = map_get(env->deopt_exit_blocks, index); + auto deopt_label = map_get(env->block_label_map, deopt_block); auto kind = instr->getInput(0)->getConstant(); arch::Gp reg = arch::reg_scratch_0; - bool is_double = false; uint64_t mask = 0; size_t sign_bit = 0; if (kind != kAlwaysFail) { - if (instr->getInput(2)->dataType() == jit::lir::OperandBase::kDouble) { - JIT_CHECK(kind == kNotZero, "Only NotZero is supported for double") - auto vecd_reg = AutoTranslator::getVecD(instr->getInput(2)); - as->umov(reg, vecd_reg); - as->cbz(reg, deopt_label); - is_double = true; + auto data_type = instr->getInput(2)->dataType(); + if (data_type == jit::lir::Operand::k8bit) { + mask = 0xFF; + sign_bit = 7; + // aarch64 doesn't have 8-bit registers, use 32-bit w register. + reg = asmjit::a64::w(instr->getInput(2)->getPhyRegister().loc); + } else if (data_type == jit::lir::Operand::k16bit) { + mask = 0xFFFF; + sign_bit = 15; + // aarch64 doesn't have 16-bit registers, use 32-bit w register. + reg = asmjit::a64::w(instr->getInput(2)->getPhyRegister().loc); } else { - auto data_type = instr->getInput(2)->dataType(); - if (data_type == jit::lir::OperandBase::k8bit) { - mask = 0xFF; - sign_bit = 7; - // aarch64 doesn't have 8-bit registers, use 32-bit w register. - reg = asmjit::a64::w(instr->getInput(2)->getPhyRegister().loc); - } else if (data_type == jit::lir::OperandBase::k16bit) { - mask = 0xFFFF; - sign_bit = 15; - // aarch64 doesn't have 16-bit registers, use 32-bit w register. - reg = asmjit::a64::w(instr->getInput(2)->getPhyRegister().loc); - } else { - reg = AutoTranslator::getGp(instr->getInput(2)); - sign_bit = reg.size() * CHAR_BIT - 1; - } + reg = AutoTranslator::getGp(instr->getInput(2)); + sign_bit = reg.size() * CHAR_BIT - 1; } } @@ -296,72 +431,60 @@ void TranslateGuard(Environ* env, const Instruction* instr) { auto target_opnd = instr->getInput(kTargetIndex); if (target_opnd->isImm() || target_opnd->isMem()) { auto target = target_opnd->getConstantOrAddress(); - JIT_DCHECK( - arm::Utils::isAddSubImm(target), - "Constant operand should fit into a 12-bit constant, optionally " - "shifted by 12 bits, got {:x}.", - target); - as->cmp(reg_arg, target); + arch::cmp_immediate(as, reg_arg, target); } else { - auto target_reg = AutoTranslator::getGp(target_opnd); + auto target_reg = AutoTranslator::getGpWiden(target_opnd); as->cmp(reg_arg, target_reg); } }; - if (!is_double) { - switch (kind) { - case kNotZero: - if (mask) { - as->tst(reg, mask); - as->b_eq(deopt_label); - } else { - as->cbz(reg, deopt_label); - } - break; - case kNotNegative: { - // Ideally we'd do but we don't know if we're outside the 32kb - // displacement limit as->tbnz(reg, sign_bit, deopt_label); - auto skip = as->newLabel(); - as->tbz(reg, sign_bit, skip); - as->b(deopt_label); - as->bind(skip); - break; + switch (kind) { + case kNotZero: + if (mask) { + as->tst(reg, mask); + as->b_eq(deopt_label); + } else { + as->cbz(reg, deopt_label); } - case kZero: - if (mask) { - as->tst(reg, mask); - as->b_ne(deopt_label); - } else { - as->cbnz(reg, deopt_label); - } - break; - case kAlwaysFail: - as->b(deopt_label); - break; - case kIs: - emit_cmp(reg); - as->b_ne(deopt_label); - break; - case kHasType: { - as->ldr( - arch::reg_scratch_0, - arch::ptr_offset(reg, offsetof(PyObject, ob_type))); - - emit_cmp(arch::reg_scratch_0); + break; + case kNotNegative: + as->tbnz(reg, sign_bit, deopt_label); + break; + case kZero: + if (mask) { + as->tst(reg, mask); as->b_ne(deopt_label); - break; + } else { + as->cbnz(reg, deopt_label); } - } + break; + case kAlwaysFail: + as->b(deopt_label); + break; + case kIs: + emit_cmp(reg); + as->b_ne(deopt_label); + break; + case kHasType: + JIT_ABORT( + "kHasType should have been lowered to kIs by postgen " + "rewriteGuardHasType"); } #else CINDER_UNSUPPORTED #endif - auto index = instr->getInput(1)->getConstant(); // skip the first four inputs in Guard, which are // kind, deopt_meta id, guard var, and target. fillLiveValueLocations(env->code_rt, index, instr, 4, instr->getNumInputs()); - env->deopt_exits.emplace_back(index, deopt_label, instr); + + // Pair this post-call guard with the preceding call's return-address label + // for the callsite->deopt-exit map used by deoptAllJitFramesOnStack(). + if (!env->pending_debug_locs.empty() && instr->origin() != nullptr && + env->pending_debug_locs.back().instr == instr->origin()) { + env->callsite_deopt_pending.emplace_back( + env->pending_debug_locs.back().label, deopt_label); + } } void TranslateDeoptPatchpoint(Environ* env, const Instruction* instr) { @@ -373,15 +496,15 @@ void TranslateDeoptPatchpoint(Environ* env, const Instruction* instr) { // Generate patchpoint by writing in an appropriately sized nop. As a future // optimization, we may be able to avoid reserving space for the patchpoint if // we can prove that the following bytes are not the target of a jump. -#if defined(CINDER_X86_64) && defined(Py_GIL_DISABLED) // On x86, align the patchpoint to 8 bytes so the patch-point doesn't straddle // a cache line boundary. This is enough to make updates appear atomic to // other cores. // // Not needed on Arm as fixed instructions are a fixed size and updates // naturally atomic. - as->align(AlignMode::kCode, 8); -#endif + if constexpr (kFreeThreadedBuild && kBuildArch == Arch::kX86_64) { + as->align(AlignMode::kCode, 8); + } auto patchpoint_label = as->newLabel(); as->bind(patchpoint_label); @@ -389,11 +512,11 @@ void TranslateDeoptPatchpoint(Environ* env, const Instruction* instr) { as->embed(stored_bytes.data(), stored_bytes.size()); // Fill in deopt metadata - auto index = instr->getInput(1)->getConstant(); + auto index = static_cast(instr->getInput(1)->getConstant()); // skip the first two inputs which are the patcher and deopt metadata id fillLiveValueLocations(env->code_rt, index, instr, 2, instr->getNumInputs()); - auto deopt_label = as->newLabel(); - env->deopt_exits.emplace_back(index, deopt_label, instr); + auto* deopt_block = map_get(env->deopt_exit_blocks, index); + auto deopt_label = map_get(env->block_label_map, deopt_block); // The runtime will link the patcher to the appropriate point in the code // once code generation has completed. @@ -404,60 +527,34 @@ void TranslateDeoptPatchpoint(Environ* env, const Instruction* instr) { void TranslateCompare(Environ* env, const Instruction* instr) { #if defined(CINDER_X86_64) auto as = env->as; - const OperandBase* inp0 = instr->getInput(0); - const OperandBase* inp1 = instr->getInput(1); + const lir::Operand* inp0 = instr->getInput(0); + const lir::Operand* inp1 = instr->getInput(1); if (inp1->isImm() || inp1->isMem()) { as->cmp(AutoTranslator::getGp(inp0), inp1->getConstantOrAddress()); } else if (!inp1->isVecD()) { as->cmp(AutoTranslator::getGp(inp0), AutoTranslator::getGp(inp1)); } else { + // Floating-point comparison; both operands are in XMM registers. `comisd` + // sets the flags in the unsigned sense (CF/ZF) and reports unordered (NaN) + // operands as CF=ZF=PF=1; the setcc below then reads those flags. + // NaN-correctness and the comparison direction are chosen when the compare + // is lowered to LIR, so a compare fused into a branch, which reuses these + // flags via compareToBranchCC on the LIR opcode, stays consistent with the + // standalone setcc emitted here. as->comisd(AutoTranslator::getVecD(inp0), AutoTranslator::getVecD(inp1)); } auto output = AutoTranslator::getGp(instr->output()); - switch (instr->opcode()) { - case Instruction::kEqual: - as->sete(output); - break; - case Instruction::kNotEqual: - as->setne(output); - break; - case Instruction::kGreaterThanSigned: - as->setg(output); - break; - case Instruction::kGreaterThanEqualSigned: - as->setge(output); - break; - case Instruction::kLessThanSigned: - as->setl(output); - break; - case Instruction::kLessThanEqualSigned: - as->setle(output); - break; - case Instruction::kGreaterThanUnsigned: - as->seta(output); - break; - case Instruction::kGreaterThanEqualUnsigned: - as->setae(output); - break; - case Instruction::kLessThanUnsigned: - as->setb(output); - break; - case Instruction::kLessThanEqualUnsigned: - as->setbe(output); - break; - default: - JIT_ABORT("bad instruction for TranslateCompare"); - } - if (instr->output()->dataType() != OperandBase::k8bit) { + as->set(asmCondCode(instr->condition()), output); + if (instr->output()->dataType() != lir::Operand::k8bit) { as->movzx( AutoTranslator::getGp(instr->output()), asmjit::x86::gpb(instr->output()->getPhyRegister().loc)); } #elif defined(CINDER_AARCH64) auto as = env->as; - const OperandBase* inp0 = instr->getInput(0); - const OperandBase* inp1 = instr->getInput(1); + const lir::Operand* inp0 = instr->getInput(0); + const lir::Operand* inp1 = instr->getInput(1); if (inp1->isMem()) { JIT_CHECK(inp1->sizeInBits() == 64, "Only 64-bit memory supported"); @@ -467,58 +564,23 @@ void TranslateCompare(Environ* env, const Instruction* instr) { as->mov(scratch, address); as->ldr(scratch, a64::ptr(scratch)); - as->cmp(AutoTranslator::getGp(inp0), scratch); + as->cmp(AutoTranslator::getGpWiden(inp0), scratch); } else if (inp1->isImm()) { auto constant = inp1->getConstantOrAddress(); - auto scratch = arch::reg_scratch_0; - - if (arm::Utils::isAddSubImm(constant)) { - as->cmp(AutoTranslator::getGp(inp0), constant); - } else { - as->mov(scratch, constant); - as->cmp(AutoTranslator::getGp(inp0), scratch); - } + arch::cmp_immediate(as, AutoTranslator::getGpWiden(inp0), constant); } else if (!inp1->isVecD()) { - as->cmp(AutoTranslator::getGp(inp0), AutoTranslator::getGp(inp1)); + as->cmp(AutoTranslator::getGpWiden(inp0), AutoTranslator::getGpWiden(inp1)); } else { + // Floating-point comparison, see the note in the x86-64 path. `fcmp` sets + // NZCV (unordered/NaN operands set C=1, V=1 while leaving Z=0), and the + // cset below reads them. NaN-correctness and the comparison + // direction are chosen when the compare is lowered to LIR, keeping the + // standalone cset and any fused b.cc consistent. as->fcmp(AutoTranslator::getVecD(inp0), AutoTranslator::getVecD(inp1)); } auto output = AutoTranslator::getGpOutput(instr->output()); - switch (instr->opcode()) { - case Instruction::kEqual: - as->cset(output, arm::CondCode::kEQ); - break; - case Instruction::kNotEqual: - as->cset(output, arm::CondCode::kNE); - break; - case Instruction::kGreaterThanSigned: - as->cset(output, arm::CondCode::kGT); - break; - case Instruction::kGreaterThanEqualSigned: - as->cset(output, arm::CondCode::kGE); - break; - case Instruction::kLessThanSigned: - as->cset(output, arm::CondCode::kLT); - break; - case Instruction::kLessThanEqualSigned: - as->cset(output, arm::CondCode::kLE); - break; - case Instruction::kGreaterThanUnsigned: - as->cset(output, arm::CondCode::kHI); - break; - case Instruction::kGreaterThanEqualUnsigned: - as->cset(output, arm::CondCode::kHS); - break; - case Instruction::kLessThanUnsigned: - as->cset(output, arm::CondCode::kLO); - break; - case Instruction::kLessThanEqualUnsigned: - as->cset(output, arm::CondCode::kLS); - break; - default: - JIT_ABORT("bad instruction for TranslateCompare"); - } + as->cset(output, asmCondCode(instr->condition())); #else CINDER_UNSUPPORTED #endif @@ -527,10 +589,10 @@ void TranslateCompare(Environ* env, const Instruction* instr) { void translateIntToBool(Environ* env, const Instruction* instr) { #if defined(CINDER_X86_64) x86::Builder* as = env->as; - const OperandBase* input = instr->getInput(0); + const lir::Operand* input = instr->getInput(0); x86::Gp output = AutoTranslator::getGp(instr->output()); JIT_CHECK( - instr->output()->dataType() == OperandBase::k8bit, + instr->output()->dataType() == lir::Operand::k8bit, "Output should be 8bits, not {}", instr->output()->dataType()); if (input->isImm()) { @@ -541,18 +603,14 @@ void translateIntToBool(Environ* env, const Instruction* instr) { } #elif defined(CINDER_AARCH64) a64::Builder* as = env->as; - const OperandBase* input = instr->getInput(0); + const lir::Operand* input = instr->getInput(0); a64::Gp output = AutoTranslator::getGpOutput(instr->output()); JIT_CHECK( - instr->output()->dataType() == OperandBase::k8bit, + instr->output()->dataType() == lir::Operand::k8bit, "Output should be 8bits, not {}", instr->output()->dataType()); - if (input->isImm()) { - as->mov(output, input->getConstant() ? 1 : 0); - } else { - as->cmp(AutoTranslator::getGp(input), 0); - as->cset(output, a64::CondCode::kNE); - } + as->cmp(AutoTranslator::getGpWiden(input), 0); + as->cset(output, a64::CondCode::kNE); #else CINDER_UNSUPPORTED #endif @@ -567,11 +625,8 @@ void emitStoreGenYieldPoint( const Instruction* yield, asmjit::Label resume_label, arch::Gp suspend_data_r, - arch::Gp scratch_r) { - bool is_yield_from = yield->isYieldFrom() || - yield->isYieldFromSkipInitialSend() || - yield->isYieldFromHandleStopAsyncIteration(); - + arch::Gp scratch_r, + bool is_yield_from) { auto calc_spill_offset = [&](size_t live_input_n) { PhyLocation mem = yield->getInput(live_input_n)->getStackSlot(); return mem.loc / kPointerSize; @@ -590,7 +645,7 @@ void emitStoreGenYieldPoint( live_regs_input); auto yield_from_offset = - is_yield_from ? calc_spill_offset(2) : kInvalidYieldFromOffset; + is_yield_from ? calc_spill_offset(0) : kInvalidYieldFromOffset; GenYieldPoint* gen_yield_point = env->code_rt->addGenYieldPoint( GenYieldPoint{deopt_idx, yield_from_offset}); @@ -616,10 +671,11 @@ void emitStoreGenYieldPoint( } void emitLoadResumedYieldInputs( - arch::Builder* as, + Environ* env, const Instruction* instr, PhyLocation sent_in_source_loc, arch::Gp tstate_reg) { + arch::Builder* as = env->as; #if defined(CINDER_X86_64) PhyLocation tstate = instr->getInput(0)->getStackSlot(); as->mov(x86::ptr(x86::rbp, tstate.loc), tstate_reg); @@ -647,17 +703,14 @@ void emitLoadResumedYieldInputs( target->type()); #elif defined(CINDER_AARCH64) PhyLocation tstate = instr->getInput(0)->getStackSlot(); - as->str( - tstate_reg, - arch::ptr_resolve(as, arch::fp, tstate.loc, arch::reg_scratch_0)); + as->str(tstate_reg, getStackSlotPtr(env, tstate.loc)); const lir::Operand* target = instr->output(); if (target->isStack()) { as->str( a64::x(sent_in_source_loc.loc), - arch::ptr_resolve( - as, arch::fp, target->getStackSlot().loc, arch::reg_scratch_0)); + getStackSlotPtr(env, target->getStackSlot().loc)); return; } @@ -678,1075 +731,860 @@ void emitLoadResumedYieldInputs( #endif } -void translateYieldInitial(Environ* env, const Instruction* instr) { -#if defined(CINDER_X86_64) -#if PY_VERSION_HEX < 0x030C0000 +void translateLoadThreadState(Environ* env, const Instruction* instr) { arch::Builder* as = env->as; + const lir::Operand* output = instr->output(); - // Load tstate into RDI for call to JITRT_MakeGenObject*. - - // Consider avoiding reloading the tstate in from memory if it was already in - // a register before spilling. Still needs to be in memory though so it can be - // recovered after calling JITRT_MakeGenObject* which will trash it. - PhyLocation tstate = instr->getInput(0)->getStackSlot(); - as->mov(x86::rdi, x86::ptr(x86::rbp, tstate.loc)); - - // Make a generator object to be returned by the epilogue. - as->lea(x86::rsi, x86::ptr(env->gen_resume_entry_label)); - JIT_CHECK( - env->shadow_frames_and_spill_size % kPointerSize == 0, - "Bad spill alignment"); - as->mov(x86::rdx, env->shadow_frames_and_spill_size / kPointerSize); - as->mov(x86::rcx, reinterpret_cast(env->code_rt)); - JIT_CHECK(instr->origin()->IsInitialYield(), "expected InitialYield"); - PyCodeObject* code = static_cast(instr->origin()) - ->frameState() - ->code; - as->mov(x86::r8, reinterpret_cast(code)); - if (code->co_flags & CO_COROUTINE) { - emitCall(*env, reinterpret_cast(JITRT_MakeGenObjectCoro), instr); - } else if (code->co_flags & CO_ASYNC_GENERATOR) { - emitCall( - *env, reinterpret_cast(JITRT_MakeGenObjectAsyncGen), instr); +#if defined(CINDER_X86_64) + x86::Gp dst; + if (output->isReg()) { + dst = x86::gpq(output->getPhyRegister().loc); + } else if (output->isStack()) { + // Use rax as scratch, will store to stack slot afterwards. + dst = x86::rax; } else { - emitCall(*env, reinterpret_cast(JITRT_MakeGenObject), instr); + JIT_ABORT("LoadThreadState output must be a register or stack slot"); } - // Resulting generator is now in RAX for filling in below and epilogue return. - const auto gen_reg = x86::rax; - - // Exit early if return from JITRT_MakeGenObject was nullptr. - as->test(gen_reg, gen_reg); - as->jz(env->hard_exit_label); - // Set RDI to gen->gi_jit_data for use in emitStoreGenYieldPoint() and data - // copy using 'movsq' below. - auto gi_jit_data_offset = offsetof(PyGenObject, gi_jit_data); - as->mov(x86::rdi, x86::ptr(gen_reg, gi_jit_data_offset)); - - // Arbitrary scratch register for use in emitStoreGenYieldPoint(). - auto scratch_r = x86::r9; - asmjit::Label resume_label = as->newLabel(); - emitStoreGenYieldPoint(as, env, instr, resume_label, x86::rdi, scratch_r); + if (cinderx::getModuleState()->tstate_offset != -1) { + // Fast path: load tstate directly from the TLS segment register. + asmjit::x86::Mem tls(cinderx::getModuleState()->tstate_offset); + tls.setSegment(x86::fs); + as->mov(dst, tls); + } else { + // Fallback: call _PyThreadState_GetCurrent(). + as->call(_PyThreadState_GetCurrent); + if (dst.id() != x86::rax.id()) { + as->mov(dst, x86::rax); + } + } - // Store variables spilled by this point to generator. - int spill_bytes = env->initial_yield_spill_size_; - JIT_CHECK(spill_bytes % kPointerSize == 0, "Bad spill alignment"); + if (output->isStack()) { + as->mov(x86::ptr(x86::rbp, output->getStackSlot().loc), dst); + } - // Point rsi at the bottom word of the current spill space. - as->lea(x86::rsi, x86::ptr(x86::rbp, -spill_bytes)); - // Point rdi at the bottom word of the generator's spill space. - as->sub(x86::rdi, spill_bytes); - as->mov(x86::rcx, spill_bytes / kPointerSize); - as->rep().movsq(); +#elif defined(CINDER_AARCH64) + a64::Gp dst; + if (output->isReg()) { + dst = a64::x(output->getPhyRegister().loc); + } else if (output->isStack()) { + dst = a64::x0; + } else { + JIT_ABORT("LoadThreadState output must be a register or stack slot"); + } - // Jump to bottom half of epilogue - as->jmp(env->hard_exit_label); + if (cinderx::getModuleState()->tstate_offset != -1) { + // Fast path: load tstate from thread-local storage. + as->mrs(dst, a64::Predicate::SysReg::kTPIDR_EL0); + as->ldr( + dst, + arch::ptr_resolve( + as, + dst, + cinderx::getModuleState()->tstate_offset, + arch::reg_scratch_0)); + } else { + // Fallback: call _PyThreadState_GetCurrent(). + as->bl(_PyThreadState_GetCurrent); + if (dst.id() != a64::x0.id()) { + as->mov(dst, a64::x0); + } + } - // Resumed execution in this generator begins here - as->bind(resume_label); + if (output->isStack()) { + as->str(dst, getStackSlotPtr(env, output->getStackSlot().loc)); + } - // Sent in value is in RSI, and tstate is in RCX from resume entry-point args - emitLoadResumedYieldInputs(as, instr, RSI, x86::rcx); #else - arch::Builder* as = env->as; - - // Load tstate into RDI for call to - // JITRT_UnlinkGenFrameAndReturnGenDataFooter. - - // Consider avoiding reloading the tstate in from memory if it was already in - // a register before spilling. Still needs to be in memory though so it can be - // recovered after calling JITRT_MakeGenObject* which will trash it. - PhyLocation tstate = instr->getInput(0)->getStackSlot(); - as->mov(x86::rdi, x86::ptr(x86::rbp, tstate.loc)); - - emitCall( - *env, - reinterpret_cast(JITRT_UnlinkGenFrameAndReturnGenDataFooter), - instr); - // This will return pointers to a generator in RAX and JIT data in RDX. + CINDER_UNSUPPORTED +#endif +} - // Arbitrary scratch register for use in emitStoreGenYieldPoint(). Any - // caller-saved register not used in this scope will do because we're on the - // exit path now. +void translateStoreGenYieldPoint(Environ* env, const Instruction* instr) { +#if defined(CINDER_X86_64) + arch::Builder* as = env->as; auto scratch_r = x86::r9; - asmjit::Label resume_label = as->newLabel(); - emitStoreGenYieldPoint(as, env, instr, resume_label, x86::rdx, scratch_r); - - // Jump to epilogue - as->jmp(env->exit_for_yield_label); - - // Resumed execution in this generator begins here - as->bind(resume_label); - - // Sent in value is in RSI, and tstate is in RCX from resume entry-point args - emitLoadResumedYieldInputs(as, instr, RSI, x86::rcx); -#endif + env->pending_yield_resume_label = as->newLabel(); + emitStoreGenYieldPoint( + as, + env, + instr, + env->pending_yield_resume_label, + x86::rbp, + scratch_r, + false); #elif defined(CINDER_AARCH64) -#if PY_VERSION_HEX < 0x030C0000 - CINDER_UNSUPPORTED -#else - arch::Builder* as = env->as; - - // Load tstate into X0 for call to - // JITRT_UnlinkGenFrameAndReturnGenDataFooter. - - // Consider avoiding reloading the tstate in from memory if it was already in - // a register before spilling. Still needs to be in memory though so it can be - // recovered after calling JITRT_MakeGenObject* which will trash it. - PhyLocation tstate = instr->getInput(0)->getStackSlot(); - as->ldr( - a64::x0, - arch::ptr_resolve(as, arch::fp, tstate.loc, arch::reg_scratch_0)); - - emitCall( - *env, - reinterpret_cast(JITRT_UnlinkGenFrameAndReturnGenDataFooter), - instr); - // This will return pointers to a generator in X0 and JIT data in X1. - - // Arbitrary scratch register for use in emitStoreGenYieldPoint(). Any - // caller-saved register not used in this scope will do because we're on the - // exit path now. + a64::Builder* as = env->as; auto scratch_r = arch::reg_scratch_0; - asmjit::Label resume_label = as->newLabel(); - emitStoreGenYieldPoint(as, env, instr, resume_label, a64::x1, scratch_r); - - // Jump to epilogue - as->b(env->exit_for_yield_label); - - // Resumed execution in this generator begins here - as->bind(resume_label); - - // Sent in value is in X1, and tstate is in X3 from resume entry-point args - emitLoadResumedYieldInputs(as, instr, X1, a64::x3); -#endif + env->pending_yield_resume_label = as->newLabel(); + emitStoreGenYieldPoint( + as, + env, + instr, + env->pending_yield_resume_label, + arch::fp, + scratch_r, + false); #else CINDER_UNSUPPORTED #endif } -void translateYieldValue(Environ* env, const Instruction* instr) { +void translateStoreGenYieldFromPoint(Environ* env, const Instruction* instr) { #if defined(CINDER_X86_64) arch::Builder* as = env->as; - - // Make sure tstate is in RDI for use in epilogue. - PhyLocation tstate = instr->getInput(0)->getStackSlot(); - as->mov(x86::rdi, x86::ptr(x86::rbp, tstate.loc)); - - // Value to send goes to RAX so it can be yielded (returned) by epilogue. - if (instr->getInput(1)->isImm()) { - as->mov(x86::rax, instr->getInput(1)->getConstant()); - } else { - PhyLocation value_out = instr->getInput(1)->getStackSlot(); - as->mov(x86::rax, x86::ptr(x86::rbp, value_out.loc)); - } - - // Arbitrary scratch register for use in emitStoreGenYieldPoint() auto scratch_r = x86::r9; - auto resume_label = as->newLabel(); - emitStoreGenYieldPoint(as, env, instr, resume_label, x86::rbp, scratch_r); + env->pending_yield_resume_label = as->newLabel(); + emitStoreGenYieldPoint( + as, + env, + instr, + env->pending_yield_resume_label, + x86::rbp, + scratch_r, + true); +#elif defined(CINDER_AARCH64) + a64::Builder* as = env->as; + auto scratch_r = arch::reg_scratch_0; + env->pending_yield_resume_label = as->newLabel(); + emitStoreGenYieldPoint( + as, + env, + instr, + env->pending_yield_resume_label, + arch::fp, + scratch_r, + true); +#else + CINDER_UNSUPPORTED +#endif +} - // Jump to epilogue - as->jmp(env->exit_for_yield_label); +void translateResumeGenYield(Environ* env, const Instruction* instr) { +#if defined(CINDER_X86_64) + arch::Builder* as = env->as; // Resumed execution in this generator begins here - as->bind(resume_label); + as->bind(env->pending_yield_resume_label); - // Sent in value is in RSI, and tstate is in RCX from resume entry-point args - emitLoadResumedYieldInputs(as, instr, RSI, x86::rcx); + // Sent in value and tstate arrive in the argument registers for the + // GenResumeFunc signature: arg[1] = sent value, arg[3] = tstate. + emitLoadResumedYieldInputs( + env, instr, ARGUMENT_REGS[1], x86::gpq(ARGUMENT_REGS[3].loc)); #elif defined(CINDER_AARCH64) a64::Builder* as = env->as; - // Make sure tstate is in x2 for use in epilogue. - PhyLocation tstate = instr->getInput(0)->getStackSlot(); - as->ldr( - a64::x2, - arch::ptr_resolve(as, arch::fp, tstate.loc, arch::reg_scratch_0)); - - // Value to send goes to x0 so it can be yielded (returned) by epilogue. - if (instr->getInput(1)->isImm()) { - as->mov(a64::x0, instr->getInput(1)->getConstant()); - } else { - PhyLocation value_out = instr->getInput(1)->getStackSlot(); - as->ldr( - a64::x0, - arch::ptr_resolve(as, arch::fp, value_out.loc, arch::reg_scratch_0)); - } - - // Arbitrary scratch register for use in emitStoreGenYieldPoint() - auto scratch_r = arch::reg_scratch_0; - auto resume_label = as->newLabel(); - emitStoreGenYieldPoint(as, env, instr, resume_label, arch::fp, scratch_r); - - // Jump to epilogue - as->b(env->exit_for_yield_label); - // Resumed execution in this generator begins here - as->bind(resume_label); + as->bind(env->pending_yield_resume_label); // Sent in value is in x1, and tstate is in x3 from resume entry-point args - emitLoadResumedYieldInputs(as, instr, X1, a64::x3); + emitLoadResumedYieldInputs(env, instr, X1, a64::x3); #else CINDER_UNSUPPORTED #endif } -void translateYieldFrom(Environ* env, const Instruction* instr) { -#if defined(CINDER_X86_64) - arch::Builder* as = env->as; - bool skip_initial_send = instr->isYieldFromSkipInitialSend(); +void translateLeaLabel(Environ* env, const Instruction* instr) { + auto* as = env->as; + auto output = instr->output(); + auto* input = instr->getInput(0); - // Make sure tstate is in RDI for use in epilogue and here. - PhyLocation tstate = instr->getInput(0)->getStackSlot(); - auto tstate_phys_reg = x86::rdi; - as->mov(tstate_phys_reg, x86::ptr(x86::rbp, tstate.loc)); - - // If we're skipping the initial send the send value is actually the first - // value to yield and so needs to go into RAX to be returned. Otherwise, - // put initial send value in RSI, the same location future send values will - // be on resume. - PhyLocation send_value = instr->getInput(1)->getStackSlot(); - const auto send_value_phys_reg = skip_initial_send ? RAX : RSI; - as->mov( - x86::gpq(send_value_phys_reg.loc), x86::ptr(x86::rbp, send_value.loc)); + JIT_CHECK(output->isReg(), "Expected output to be a register"); + JIT_CHECK(input->isLabel(), "Expected input to be a label"); - asmjit::Label yield_label = as->newLabel(); - if (skip_initial_send) { - as->jmp(yield_label); - } else { - // Setup call to JITRT_GenSend - - // Put tstate and the current generator into RCX and RDI respectively, and - // set finish_yield_from (RDX) to 0. This register setup matches that when - // `resume_label` is reached from the resume entry. - auto gen_offs = offsetof(GenDataFooter, gen); - as->mov(x86::rcx, tstate_phys_reg); - as->mov(x86::rdi, x86::ptr(x86::rbp, gen_offs)); - as->xor_(x86::rdx, x86::rdx); - } - - // Resumed execution begins here - auto resume_label = as->newLabel(); - as->bind(resume_label); - - // Save tstate from resume to callee-saved reigster. - as->mov(x86::rbx, x86::rcx); - - // 'send_value', and 'finish_yield_from' will already be in RSI and RCX - // respectively, either from code above on initial start or from resume entry - // point args. - - // Load sub-iterator into RDI - PhyLocation iter_slot = instr->getInput(2)->getStackSlot(); - as->mov(x86::rdi, x86::ptr(x86::rbp, iter_slot.loc)); - - uint64_t func = reinterpret_cast( - instr->isYieldFromHandleStopAsyncIteration() - ? JITRT_GenSendHandleStopAsyncIteration - : JITRT_GenSend); - emitCall(*env, func, instr); - // Yielded or final result value now in RAX. If the result was nullptr then - // done will be set so we'll correctly jump to the following CheckExc. - const auto yf_result_phys_reg = RAX; - const auto done_r = x86::rdx; - - // Restore tstate from callee-saved register. - as->mov(tstate_phys_reg, x86::rbx); - - // If not done, jump to epilogue which will yield/return the value from - // JITRT_GenSend in RAX. - as->test(done_r, done_r); - asmjit::Label done_label = as->newLabel(); - as->jnz(done_label); - - as->bind(yield_label); - // Arbitrary scratch register for use in emitStoreGenYieldPoint() - auto scratch_r = x86::r9; - emitStoreGenYieldPoint(as, env, instr, resume_label, x86::rbp, scratch_r); - as->jmp(env->exit_for_yield_label); + asmjit::Label label = input->getDefine()->hasAsmLabel() + ? input->getDefine()->getAsmLabel() + : map_get(env->block_label_map, input->getBasicBlock()); - as->bind(done_label); - emitLoadResumedYieldInputs(as, instr, yf_result_phys_reg, tstate_phys_reg); +#if defined(CINDER_X86_64) + as->lea(x86::gpq(output->getPhyRegister().loc), x86::ptr(label)); #elif defined(CINDER_AARCH64) - arch::Builder* as = env->as; - bool skip_initial_send = instr->isYieldFromSkipInitialSend(); + as->adr(a64::x(output->getPhyRegister().loc), label); +#else + CINDER_UNSUPPORTED +#endif +} - // Make sure tstate is in X0 for use in epilogue and here. - PhyLocation tstate = instr->getInput(0)->getStackSlot(); - auto tstate_phys_reg = a64::x0; - as->ldr( - tstate_phys_reg, - arch::ptr_resolve(as, arch::fp, tstate.loc, arch::reg_scratch_0)); - - // If we're skipping the initial send the send value is actually the first - // value to yield and so needs to go into X0 to be returned. Otherwise, - // put initial send value in X1, the same location future send values will - // be on resume. - PhyLocation send_value = instr->getInput(1)->getStackSlot(); - const auto send_value_phys_reg = skip_initial_send ? X0 : X1; - as->ldr( - a64::x(send_value_phys_reg.loc), - arch::ptr_resolve(as, arch::fp, send_value.loc, arch::reg_scratch_0)); - - asmjit::Label yield_label = as->newLabel(); - if (skip_initial_send) { - as->b(yield_label); - } else { - // Setup call to JITRT_GenSend - - // Put tstate and the current generator into X3 and X0 respectively, and - // set finish_yield_from (X2) to 0. This register setup matches that when - // `resume_label` is reached from the resume entry. - auto gen_offs = offsetof(GenDataFooter, gen); - as->mov(a64::x3, tstate_phys_reg); - as->ldr(a64::x0, arch::ptr_offset(arch::fp, gen_offs)); - as->mov(a64::x2, a64::xzr); - } - - // Resumed execution begins here - auto resume_label = as->newLabel(); - as->bind(resume_label); - - // Save tstate from resume to callee-saved reigster. - as->mov(a64::x19, a64::x3); - - // 'send_value', and 'finish_yield_from' will already be in X1 and X3 - // respectively, either from code above on initial start or from resume entry - // point args. - - // Load sub-iterator into X0 - PhyLocation iter_slot = instr->getInput(2)->getStackSlot(); - as->ldr( - a64::x0, - arch::ptr_resolve(as, arch::fp, iter_slot.loc, arch::reg_scratch_0)); - - uint64_t func = reinterpret_cast( - instr->isYieldFromHandleStopAsyncIteration() - ? JITRT_GenSendHandleStopAsyncIteration - : JITRT_GenSend); - emitCall(*env, func, instr); - // Yielded or final result value now in X0. If the result was nullptr then - // done will be set so we'll correctly jump to the following CheckExc. - const auto yf_result_phys_reg = X0; - const auto done_r = a64::x2; - - // Restore tstate from callee-saved register. - as->mov(tstate_phys_reg, a64::x19); - - // If not done, jump to epilogue which will yield/return the value from - // JITRT_GenSend in X0. - asmjit::Label done_label = as->newLabel(); - as->cbnz(done_r, done_label); - - as->bind(yield_label); - // Arbitrary scratch register for use in emitStoreGenYieldPoint() - auto scratch_r = arch::reg_scratch_0; - emitStoreGenYieldPoint(as, env, instr, resume_label, arch::fp, scratch_r); - as->b(env->exit_for_yield_label); +// Lower LIR ReserveStack to a LEA (x86-64) or ADD (aarch64) that computes +// the address of the reserved stack region. The reserved data lives at +// [SP + max_arg_buffer_size], above the call argument buffer, so that +// function calls don't clobber it. +void translateReserveStack(Environ* env, const Instruction* instr) { + auto* as = env->as; + auto output = instr->output(); + JIT_CHECK(output->isReg(), "Expected output to be a register"); + + int offset = env->max_arg_buffer_size; - as->bind(done_label); - emitLoadResumedYieldInputs(as, instr, yf_result_phys_reg, tstate_phys_reg); +#if defined(CINDER_X86_64) + as->lea(x86::gpq(output->getPhyRegister().loc), x86::ptr(x86::rsp, offset)); +#elif defined(CINDER_AARCH64) + arch::add_signed_immediate( + as, a64::x(output->getPhyRegister().loc), a64::sp, offset); #else CINDER_UNSUPPORTED #endif } -// *********************************************************************** -// The following templates and macros implement the auto generation table. -// The generator table defines a hash table, whose key is instruction type, -// and value is another hash table mapping instruction operand pattern and -// a function carrying out certain Actions for the instruction with the -// operand pattern. -// The list of Actions are encoded in the template class RuleActions as its -// template arguments. Currently, there are two types of Actions: -// * AsmAction - generate an asm instruction -// * CallAction - call a user defined instruction -// The Action classes are also templates, whose argument lists encode the -// parameters for the Action. For example, an AsmAction's argument list has -// the assembly instruction mnemonic and its operands. -// *********************************************************************** -template -const OperandBase* LIROperandMapper(const Instruction* instr) { - auto num_outputs = instr->getNumOutputs(); - if (N < num_outputs) { - return instr->output(); +void translateEpilogueEnd(Environ* env, const Instruction* instr) { + auto* as = env->as; + + auto* ret_val = instr->getInput(0); + bool is_primitive = ret_val->dataType() != DataType::kObject && + ret_val->dataType() != DataType::kObjectUntagged; + bool is_double = ret_val->isFp(); + +#if defined(CINDER_X86_64) + // Move return value to ABI return register + if (is_double) { + if (ret_val->isStack()) { + as->movsd(x86::xmm0, x86::ptr(x86::rbp, ret_val->getStackSlot().loc)); + } else if ( + ret_val->isReg() && + ret_val->getPhyRegister().loc != arch::reg_double_return_loc.loc) { + as->movsd( + x86::xmm0, x86::xmm(ret_val->getPhyRegister().loc - VECD_REG_BASE)); + } } else { - return instr->getInput(N - num_outputs); + if (ret_val->isStack()) { + as->mov(x86::rax, x86::ptr(x86::rbp, ret_val->getStackSlot().loc)); + } else if ( + ret_val->isReg() && + ret_val->getPhyRegister().loc != arch::reg_general_return_loc.loc) { + as->mov(x86::rax, x86::gpq(ret_val->getPhyRegister().loc)); + } } -} -template -int LIROperandSizeMapper(const Instruction* instr) { - auto size_type = InstrProperty::getProperties(instr).opnd_size_type; - switch (size_type) { - case kDefault: - return LIROperandMapper(instr)->sizeInBits(); - case kAlways64: - return 64; - case kOut: - return LIROperandMapper<0>(instr)->sizeInBits(); + if (is_primitive) { + if (is_double) { + as->pcmpeqw(x86::xmm1, x86::xmm1); + as->psrlq(x86::xmm1, 63); + } else { + as->mov(x86::edx, 1); + } } - JIT_ABORT("Unknown size type"); -} - -template -struct ImmOperand { - using asmjit_type = const asmjit::Imm&; - - static asmjit::Imm GetAsmOperand(Environ*, const Instruction* instr) { - return asmjit::Imm(LIROperandMapper(instr)->getConstant()); + as->bind(env->hard_exit_label); + auto saved_regs = env->changed_regs & CALLEE_SAVE_REGS; + if (!saved_regs.empty()) { + JIT_CHECK( + env->last_callee_saved_reg_off != -1, + "offset to callee saved regs not initialized"); + // Point rsp at the bottom of the callee-saved area. + as->lea(x86::rsp, x86::ptr(x86::rbp, -env->last_callee_saved_reg_off)); +#ifdef _WIN32 + // On Windows, callee-saved XMM registers were saved with movaps and + // must be restored the same way. GP registers are restored with pop. + auto vecd_regs = saved_regs & ALL_VECD_REGISTERS; + auto gp_regs = saved_regs & ALL_GP_REGISTERS; + int xmm_offset = 0; + while (!vecd_regs.empty()) { + auto reg = vecd_regs.getFirst(); + as->movaps( + x86::xmm(reg.loc - VECD_REG_BASE), x86::ptr(x86::rsp, xmm_offset)); + xmm_offset += kVecDSize; + vecd_regs.removeFirst(); + } + int vecd_count = (saved_regs & ALL_VECD_REGISTERS).count(); + int gp_count = gp_regs.count(); + int vecd_area_size = vecd_count * kVecDSize; + if (vecd_count > 0 && (gp_count * kPointerSize) % kStackAlign != 0) { + vecd_area_size += kPointerSize; + } + if (vecd_area_size > 0) { + as->add(x86::rsp, vecd_area_size); + } + while (!gp_regs.empty()) { + as->pop(x86::gpq(gp_regs.getLast().loc)); + gp_regs.removeLast(); + } +#else + // Pop in reverse push order (GetLast→GetFirst) to restore registers. + while (!saved_regs.empty()) { + as->pop(x86::gpq(saved_regs.getLast().loc)); + saved_regs.removeLast(); + } +#endif + } + as->leave(); + as->ret(); +#elif defined(CINDER_AARCH64) + // Move return value to ABI return register + if (is_double) { + if (ret_val->isStack()) { + as->ldr(a64::d0, getStackSlotPtr(env, ret_val->getStackSlot().loc)); + } else if ( + ret_val->isReg() && + ret_val->getPhyRegister().loc != arch::reg_double_return_loc.loc) { + as->fmov(a64::d0, a64::d(ret_val->getPhyRegister().loc - VECD_REG_BASE)); + } + } else { + if (ret_val->isStack()) { + as->ldr(a64::x0, getStackSlotPtr(env, ret_val->getStackSlot().loc)); + } else if ( + ret_val->isReg() && + ret_val->getPhyRegister().loc != arch::reg_general_return_loc.loc) { + as->mov(a64::x0, a64::x(ret_val->getPhyRegister().loc)); + } } -}; - -template -struct ImmOperandNegate { - using asmjit_type = const asmjit::Imm&; - static asmjit::Imm GetAsmOperand(Environ* env, const Instruction* instr) { - return asmjit::Imm( - -T::GetAsmOperand(env, instr).template valueAs()); + if (is_primitive) { + if (is_double) { + as->fmov(a64::d1, 1.0); + } else { + as->mov(a64::w1, 1); + } } -}; -template -struct ImmOperandInvert { - using asmjit_type = const asmjit::Imm&; + as->bind(env->hard_exit_label); + auto saved_regs = env->changed_regs & CALLEE_SAVE_REGS; + if (!saved_regs.empty()) { + JIT_CHECK( + env->last_callee_saved_reg_off != -1, + "offset to callee saved regs not initialized"); + JIT_CHECK(env->last_callee_saved_reg_off % kStackAlign == 0, "unaligned"); + // Restore callee-saved registers from fixed offsets below FP. + // Use a scratch register as base to avoid large FP-relative offsets + // that can exceed arm64 ldp/ldr encoding range. + auto gp_regs = saved_regs & ALL_GP_REGISTERS; + auto vecd_regs = saved_regs & ALL_VECD_REGISTERS; + + int gp_size = (((gp_regs.count() + 1) / 2)) * kStackAlign; + int vecd_size = (((vecd_regs.count() + 1) / 2)) * kStackAlign; + int header_and_spill_size = + env->last_callee_saved_reg_off - gp_size - vecd_size; + + // base = fp - header_and_spill_size (points to start of callee-saved area) + arch::sub_immediate( + as, + arch::reg_scratch_0, + arch::fp, + static_cast(header_and_spill_size)); + + // Restore GP registers (iterate GetFirst→GetLast, same as save). + int offset = 0; + if (!gp_regs.empty()) { + if (gp_regs.count() % 2 == 1) { + as->ldr( + a64::x(gp_regs.getFirst().loc), + a64::ptr(arch::reg_scratch_0, -(offset + 16))); + gp_regs.removeFirst(); + offset += 16; + } + while (!gp_regs.empty()) { + auto first = a64::x(gp_regs.getFirst().loc); + gp_regs.removeFirst(); + auto second = a64::x(gp_regs.getFirst().loc); + gp_regs.removeFirst(); + as->ldp(first, second, a64::ptr(arch::reg_scratch_0, -(offset + 16))); + offset += 16; + } + } - static asmjit::Imm GetAsmOperand(Environ* env, const Instruction* instr) { - return asmjit::Imm( - ~T::GetAsmOperand(env, instr).template valueAs()); + // Restore VecD registers (iterate GetFirst→GetLast, same as save). + if (!vecd_regs.empty()) { + if (vecd_regs.count() % 2 == 1) { + as->ldr( + a64::d(vecd_regs.getFirst().loc - VECD_REG_BASE), + a64::ptr(arch::reg_scratch_0, -(offset + 16))); + vecd_regs.removeFirst(); + offset += 16; + } + while (!vecd_regs.empty()) { + auto first = a64::d(vecd_regs.getFirst().loc - VECD_REG_BASE); + vecd_regs.removeFirst(); + auto second = a64::d(vecd_regs.getFirst().loc - VECD_REG_BASE); + vecd_regs.removeFirst(); + as->ldp(first, second, a64::ptr(arch::reg_scratch_0, -(offset + 16))); + offset += 16; + } + } } -}; - -template -struct RegOperand { - using asmjit_type = const arch::Gp&; - static arch::Gp GetAsmOperand(Environ*, const Instruction* instr) { - static_assert( - Size == -1 || Size == 8 || Size == 16 || Size == 32 || Size == 64, - "Invalid Size"); + as->mov(a64::sp, arch::fp); + as->ldp(arch::fp, arch::lr, a64::ptr_post(a64::sp, arch::kFrameRecordSize)); + as->ret(arch::lr); + env->sp_to_fp_delta = arch::kSpPositionUnknown; +#else + CINDER_UNSUPPORTED +#endif +} +// Emit the function entry sequence (push frame pointer, set up new frame). +void translatePrologue(Environ* env, const Instruction*) { + arch::Builder* as = env->as; + asmjit::BaseNode* cursor = as->cursor(); #if defined(CINDER_X86_64) - int size = Size == -1 ? LIROperandSizeMapper(instr) : Size; - - PhyLocation reg = LIROperandMapper(instr)->getPhyRegister(); - switch (size) { - case 8: - return asmjit::x86::gpb(reg.loc); - case 16: - return asmjit::x86::gpw(reg.loc); - case 32: - return asmjit::x86::gpd(reg.loc); - case 64: - return asmjit::x86::gpq(reg.loc); - } + as->push(x86::rbp); + as->mov(x86::rbp, x86::rsp); #elif defined(CINDER_AARCH64) - int size = Size == -1 ? LIROperandSizeMapper(instr) : Size; + as->stp(arch::fp, arch::lr, a64::ptr_pre(a64::sp, -arch::kFrameRecordSize)); + as->mov(arch::fp, a64::sp); +#else + CINDER_UNSUPPORTED +#endif + env->addAnnotation(std::string("Set up frame pointer"), cursor); +} - PhyLocation reg = LIROperandMapper(instr)->getPhyRegister(); - switch (size) { - case 8: - case 16: - JIT_ABORT("Currently unsupported size."); - case 32: - return asmjit::a64::w(reg.loc); - case 64: - return asmjit::a64::x(reg.loc); +// Allocate the full stack frame and save callee-saved registers. +// All frame layout values come from Environ, populated after register +// allocation. +void translateSetupFrame(Environ* env, const Instruction*) { + arch::Builder* as = env->as; + +#if defined(CINDER_X86_64) + // Allocate header + spill space, then save callee-saved registers. + asmjit::BaseNode* alloc_cursor = as->cursor(); + as->sub(x86::rsp, env->resume_header_and_spill_size); + env->addAnnotation(std::string("Allocate stack frame"), alloc_cursor); + + asmjit::BaseNode* save_cursor = as->cursor(); + auto gp_saved_regs = env->resume_saved_regs & ALL_GP_REGISTERS; + // Push GP callee-saved registers (1-2B per register). + while (!gp_saved_regs.empty()) { + as->push(x86::gpq(gp_saved_regs.getFirst().loc)); + gp_saved_regs.removeFirst(); + } + + auto gp_save_count = (env->resume_saved_regs & ALL_GP_REGISTERS).count(); +#ifdef _WIN32 + auto vecd_saved_regs = env->resume_saved_regs & ALL_VECD_REGISTERS; + auto vecd_save_count = vecd_saved_regs.count(); + + // On Windows, callee-saved XMM registers (XMM6-XMM15) are saved via movaps + // into the stack space between the GP pushes and the arg buffer. + // Compute the offset where XMM saves start (right after GP pushes, aligned). + int vecd_area_size = vecd_save_count * kVecDSize; + if (vecd_save_count > 0 && + (gp_save_count * kPointerSize) % kStackAlign != 0) { + vecd_area_size += kPointerSize; // alignment padding + } + int arg_buffer_size = env->resume_frame_total_size - + env->resume_header_and_spill_size - gp_save_count * kPointerSize - + vecd_area_size; + if (vecd_area_size + arg_buffer_size > 0) { + as->sub(x86::rsp, vecd_area_size + arg_buffer_size); + } + // Save XMM registers into [rsp + arg_buffer_size + offset] + int xmm_offset = arg_buffer_size; + while (!vecd_saved_regs.empty()) { + auto reg = vecd_saved_regs.getFirst(); + as->movaps( + x86::ptr(x86::rsp, xmm_offset), x86::xmm(reg.loc - VECD_REG_BASE)); + xmm_offset += kVecDSize; + vecd_saved_regs.removeFirst(); + } +#else + int arg_buffer_size = env->resume_frame_total_size - + env->resume_header_and_spill_size - gp_save_count * kPointerSize; + if (arg_buffer_size > 0) { + as->sub(x86::rsp, arg_buffer_size); + } +#endif + env->addAnnotation(std::string("Save callee-saved registers"), save_cursor); +#elif defined(CINDER_AARCH64) + // allocateHeaderAndSpillSpace() + asmjit::BaseNode* alloc_cursor = as->cursor(); + arch::sub_immediate( + as, + a64::sp, + a64::sp, + static_cast(env->resume_frame_total_size)); + // A generator's body runs with FP pointing at its heap-allocated + // GenDataFooter rather than at the machine stack (see + // LIRGenerator::emitLoadFrame), so there is no delta to track and SP-relative + // frame slots would address unrelated memory. The FP swap already invalidates + // the delta via the SP/FP write guard in AutoTranslator::translateInstr, but + // never establish one in the first place so that a future change which + // re-establishes it mid-body can't silently resurrect the hazard. + env->sp_to_fp_delta = env->is_generator ? arch::kSpPositionUnknown + : env->resume_frame_total_size; + env->addAnnotation(std::string("Allocate stack frame"), alloc_cursor); + + // saveCallerRegisters() + asmjit::BaseNode* save_cursor = as->cursor(); + auto gp_regs = env->resume_saved_regs & ALL_GP_REGISTERS; + auto vecd_regs = env->resume_saved_regs & ALL_VECD_REGISTERS; + + arch::sub_immediate( + as, + arch::reg_scratch_0, + arch::fp, + static_cast(env->resume_header_and_spill_size)); + + int reg_offset = 0; + if (!gp_regs.empty()) { + if (gp_regs.count() % 2 == 1) { + as->str( + a64::x(gp_regs.getFirst().loc), + a64::ptr(arch::reg_scratch_0, -(reg_offset + 16))); + gp_regs.removeFirst(); + reg_offset += 16; + } + while (!gp_regs.empty()) { + auto first = a64::x(gp_regs.getFirst().loc); + gp_regs.removeFirst(); + auto second = a64::x(gp_regs.getFirst().loc); + gp_regs.removeFirst(); + as->stp(first, second, a64::ptr(arch::reg_scratch_0, -(reg_offset + 16))); + reg_offset += 16; + } + } + if (!vecd_regs.empty()) { + if (vecd_regs.count() % 2 == 1) { + as->str( + a64::d(vecd_regs.getFirst().loc - VECD_REG_BASE), + a64::ptr(arch::reg_scratch_0, -(reg_offset + 16))); + vecd_regs.removeFirst(); + reg_offset += 16; + } + while (!vecd_regs.empty()) { + auto first = a64::d(vecd_regs.getFirst().loc - VECD_REG_BASE); + vecd_regs.removeFirst(); + auto second = a64::d(vecd_regs.getFirst().loc - VECD_REG_BASE); + vecd_regs.removeFirst(); + as->stp(first, second, a64::ptr(arch::reg_scratch_0, -(reg_offset + 16))); + reg_offset += 16; } + } + env->addAnnotation(std::string("Save callee-saved registers"), save_cursor); #else - CINDER_UNSUPPORTED + CINDER_UNSUPPORTED #endif +} - JIT_ABORT("Incorrect operand size."); - } -}; +// Emit a branch through a memory-indirect operand [base + offset]. +// Used by kBranch when its input is a MemoryIndirect operand. +void translateBranchIndirect(Environ* env, const Instruction* instr) { + arch::Builder* as = env->as; + const lir::Operand* input = instr->getInput(0); -template -struct VecDOperand { - using asmjit_type = const arch::VecD&; - static arch::VecD GetAsmOperand(Environ*, const Instruction* instr) { + if (input->isReg()) { #if defined(CINDER_X86_64) - return asmjit::x86::xmm( - LIROperandMapper(instr)->getPhyRegister().loc - VECD_REG_BASE); + as->jmp(AutoTranslator::getGp(input)); #elif defined(CINDER_AARCH64) - return asmjit::a64::d( - LIROperandMapper(instr)->getPhyRegister().loc - VECD_REG_BASE); + as->br(AutoTranslator::getGp(input)); #else CINDER_UNSUPPORTED - return arch::VecD(); #endif + return; } -}; - -#define OP(v) \ - typename std::conditional_t< \ - pattern[v] == 'i', \ - ImmOperand, \ - std::conditional_t< \ - (pattern[v] == 'x' || pattern[v] == 'X'), \ - VecDOperand, \ - RegOperand>> -#define REG_OP(v, size) RegOperand + JIT_CHECK( + input->isInd(), + "Branch indirect input must be memory indirect or register"); -arch::Mem AsmIndirectOperandBuilder(const OperandBase* operand) { - JIT_DCHECK(operand->isInd(), "operand should be an indirect reference"); + const auto* mem = input->getMemoryIndirect(); + PhyLocation base = mem->getBaseRegOperand()->getPhyRegister(); + int32_t disp = mem->getOffset(); #if defined(CINDER_X86_64) - auto indirect = operand->getMemoryIndirect(); - - OperandBase* base = indirect->getBaseRegOperand(); - OperandBase* index = indirect->getIndexRegOperand(); - - if (index == nullptr) { - return asmjit::x86::ptr( - x86::gpq(base->getPhyRegister().loc), indirect->getOffset()); - } else { - return asmjit::x86::ptr( - x86::gpq(base->getPhyRegister().loc), - x86::gpq(index->getPhyRegister().loc), - indirect->getMultipiler(), - indirect->getOffset()); - } + as->jmp(x86::ptr(x86::gpq(base.loc), disp)); #elif defined(CINDER_AARCH64) - JIT_ABORT("Unreachable."); + auto ptr = arch::ptr_resolve(as, a64::x(base.loc), disp, arch::reg_scratch_0); + as->ldr(arch::reg_scratch_br, ptr); + as->br(arch::reg_scratch_br); #else CINDER_UNSUPPORTED - return arch::Mem(); #endif } -template -struct MemOperand { - using asmjit_type = const arch::Mem&; - static arch::Mem GetAsmOperand(Environ*, const Instruction* instr) { -#if defined(CINDER_X86_64) - const OperandBase* operand = LIROperandMapper(instr); - auto size = LIROperandSizeMapper(instr) / 8; - - asmjit::x86::Mem memptr; - if (operand->isStack()) { - memptr = asmjit::x86::ptr(asmjit::x86::rbp, operand->getStackSlot().loc); - } else if (operand->isMem()) { - memptr = asmjit::x86::ptr( - reinterpret_cast(operand->getMemoryAddress())); - } else if (operand->isInd()) { - memptr = AsmIndirectOperandBuilder(operand); - } else { - JIT_ABORT("Unsupported operand type."); - } +// Emit a variadic sequence of GP register pushes (x86) or stp pairs (aarch64). +// Each input operand is a physical register to save. The registers are stored +// in input order (first input is the lowest address). +void translateVariadicPush(Environ* env, const Instruction* instr) { + arch::Builder* as = env->as; + size_t n = instr->getNumInputs(); - memptr.setSize(size); - return memptr; +#if defined(CINDER_X86_64) + for (size_t i = 0; i < n; i++) { + as->push(x86::gpq(instr->getInput(n - i - 1)->getPhyRegister().loc)); + } #elif defined(CINDER_AARCH64) - const OperandBase* operand = LIROperandMapper(instr); - if (!operand->isStack()) { - JIT_ABORT("Unreachable."); - } - - int32_t loc = operand->getStackSlot().loc; - JIT_CHECK(loc >= -256 && loc < 256, "Stack slot out of range"); + // First pair uses pre-index to allocate stack space for all pairs. + // Remaining pairs use offset addressing within the allocated region. + constexpr int bytes_per_store = 16; + int total_pairs = (n + 1) / 2; + int alloc = total_pairs * bytes_per_store; + size_t i = 0; + + // First pair: pre-index allocation + if (n >= 2) { + as->stp( + a64::x(instr->getInput(0)->getPhyRegister().loc), + a64::x(instr->getInput(1)->getPhyRegister().loc), + a64::ptr_pre(a64::sp, -alloc)); + i = 2; + } else if (n == 1) { + as->str( + a64::x(instr->getInput(0)->getPhyRegister().loc), + a64::ptr_pre(a64::sp, -alloc)); + i = 1; + } + + // Remaining pairs at positive offsets from sp. + int pair_idx = 1; + while (i + 1 < n) { + as->stp( + a64::x(instr->getInput(i)->getPhyRegister().loc), + a64::x(instr->getInput(i + 1)->getPhyRegister().loc), + a64::ptr(a64::sp, pair_idx * bytes_per_store)); + i += 2; + pair_idx++; + } + if (i < n) { + as->str( + a64::x(instr->getInput(i)->getPhyRegister().loc), + a64::ptr(a64::sp, pair_idx * bytes_per_store)); + } - return arch::ptr_offset(arch::fp, loc); + env->adjustSp(alloc); #else - CINDER_UNSUPPORTED - return arch::Mem(); + CINDER_UNSUPPORTED #endif - } -}; +} -#define MEM(m) MemOperand -#define STK(v) MemOperand +// Store a pair of GP register values at consecutive pointer-sized slots. +// Input 0: immediate offset. Input 1: base register. +// Inputs 2, 3: values stored at [base+offset] and [base+offset+8]. +#if defined(CINDER_AARCH64) + +// Resolve the address of a load/store pair, preferring the single-instruction +// stp/ldp form. Returns nullopt when the pair can't be encoded and the caller +// has to fall back to two separate accesses. +// +// A frame-pointer-relative pair is also reachable from SP while the frame is +// established (see getStackSlotPtr), and the two bases have very different +// reach: the scaled 7-bit offset covers -512..504 either way, but the SP form +// measures from the other end of the frame, so one can encode where the other +// can't. Both are tried before giving up. +std::optional +getPairPtr(Environ* env, PhyLocation base_reg, int32_t offset) { + auto encodable = [](int32_t off) { + return (off & (kPointerSize - 1)) == 0 && Support::isInt7(off >> 3); + }; -template -struct LabelOperand { - using asmjit_type = const asmjit::Label&; - static asmjit::Label GetAsmOperand(Environ* env, const Instruction* instr) { - auto block = LIROperandMapper(instr)->getBasicBlock(); - return map_get(env->block_label_map, block); + if (base_reg == arch::reg_frame_pointer_loc) { + if (env->sp_to_fp_delta != arch::kSpPositionUnknown) { + // pairMemoryLocation() reports the same base for real frame slots and for + // genuine FP-relative indirects (which a generator uses to reach its + // GenDataFooter). Only the former may be rewritten, and the delta is only + // ever known while FP is a real frame pointer, so assert that here. + JIT_DCHECK( + offset < 0, + "Frame slot offsets must be negative FP offsets, got {}", + offset); + int32_t sp_offset = offset + env->sp_to_fp_delta; + JIT_DCHECK( + sp_offset >= 0, + "SP-relative frame slot at {} must not be below SP (delta {})", + offset, + env->sp_to_fp_delta); + if (encodable(sp_offset)) { + return a64::ptr(a64::sp, sp_offset); + } + } } -}; -#define LBL(v) LabelOperand + if (encodable(offset)) { + auto base = base_reg == SP ? a64::sp : a64::x(base_reg.loc); + return a64::ptr(base, offset); + } -template -struct OperandList; + return std::nullopt; +} -template -struct AsmAction; +// Materialize base+offset in the scratch register so a pair whose offset is +// out of stp/ldp range can still be issued as a single instruction from +// [scratch]. +// +// The two halves must not be resolved separately: each resolution recomputes +// an address into the same scratch, so the second one destroys a pair register +// that happens to be that scratch. A store would then write the address +// instead of its value, and a load would lose the value it had just read. +// pairAdjacentMemoryOps refuses to build a pair that would land here holding a +// scratch register, so the check below is a tripwire rather than a live case. +asmjit::a64::Mem getPairScratchPtr( + Environ* env, + PhyLocation base_reg, + int32_t offset, + PhyLocation reg0, + PhyLocation reg1) { + JIT_CHECK( + reg0 != arch::reg_scratch_0_loc && reg1 != arch::reg_scratch_0_loc, + "pair at offset {} holds the address scratch {} in a value/destination " + "slot", + offset, + arch::reg_scratch_0_loc); + auto base = base_reg == SP ? a64::sp : a64::x(base_reg.loc); + arch::add_signed_immediate(env->as, arch::reg_scratch_0, base, offset); + return a64::ptr(arch::reg_scratch_0); +} -template -struct AsmAction> { - static void eval(Environ* env, const Instruction* instr) { - static_cast(instr); - (env->as->*func)(OpndTypes::GetAsmOperand(env, instr)...); - } -}; +#endif -template -struct AsminstructionType { - using type = asmjit::Error (arch::EmitterExplicitT::*)( - typename Args::asmjit_type...); -}; +void translateStorePair(Environ* env, const Instruction* instr) { + arch::Builder* as = env->as; + JIT_DCHECK( + instr->getNumInputs() == 4, + "StorePair expects exactly 4 inputs (offset, base, val0, val1)"); + int32_t offset = static_cast(instr->getInput(0)->getConstant()); -template -struct CallAction { - static void eval(Environ* env, const Instruction* instr) { - func(env, instr); +#if defined(CINDER_X86_64) + auto base = x86::gpq(instr->getInput(1)->getPhyRegister().loc); + as->mov( + x86::qword_ptr(base, offset), + x86::gpq(instr->getInput(2)->getPhyRegister().loc)); + as->mov( + x86::qword_ptr(base, offset + kPointerSize), + x86::gpq(instr->getInput(3)->getPhyRegister().loc)); +#elif defined(CINDER_AARCH64) + auto base_reg = instr->getInput(1)->getPhyRegister(); + auto val0_loc = instr->getInput(2)->getPhyRegister(); + auto val1_loc = instr->getInput(3)->getPhyRegister(); + auto val0 = a64::x(val0_loc.loc); + auto val1 = a64::x(val1_loc.loc); + + if (auto ptr = getPairPtr(env, base_reg, offset)) { + as->stp(val0, val1, *ptr); + } else { + as->stp( + val0, + val1, + getPairScratchPtr(env, base_reg, offset, val0_loc, val1_loc)); } -}; - -template -struct RuleActions; +#else + CINDER_UNSUPPORTED +#endif +} -template -struct RuleActions { - static void eval(Environ* env, const Instruction* instr) { - AAction::eval(env, instr); - RuleActions::eval(env, instr); - } -}; +void translateLoadPair(Environ* env, const Instruction* instr) { + arch::Builder* as = env->as; + JIT_DCHECK( + instr->getNumInputs() == 3, + "LoadPair expects exactly 3 inputs (offset, base, dst1)"); + int32_t offset = static_cast(instr->getInput(0)->getConstant()); + auto base_reg = instr->getInput(1)->getPhyRegister(); + auto dst0_loc = instr->output()->getPhyRegister(); + auto dst1_loc = instr->getInput(2)->getPhyRegister(); -template <> -struct RuleActions<> { - static void eval(Environ*, const Instruction*) {} -}; +#if defined(CINDER_X86_64) + auto base = x86::gpq(base_reg.loc); + as->mov(x86::gpq(dst0_loc.loc), x86::qword_ptr(base, offset)); + as->mov(x86::gpq(dst1_loc.loc), x86::qword_ptr(base, offset + kPointerSize)); +#elif defined(CINDER_AARCH64) + auto dst0 = a64::x(dst0_loc.loc); + auto dst1 = a64::x(dst1_loc.loc); -struct AddDebugEntryAction { - static void eval(Environ* env, const Instruction* instr) { - asmjit::Label label = env->as->newLabel(); - env->as->bind(label); - if (instr->origin()) { - env->pending_debug_locs.emplace_back(label, instr->origin()); - } + if (auto ptr = getPairPtr(env, base_reg, offset)) { + as->ldp(dst0, dst1, *ptr); + } else { + as->ldp( + dst0, + dst1, + getPairScratchPtr(env, base_reg, offset, dst0_loc, dst1_loc)); } -}; - -} // namespace +#else + CINDER_UNSUPPORTED +#endif +} -#define ASM(instr, args...) \ - AsmAction< \ - typename AsminstructionType::type, \ - &arch::Builder::instr, \ - OperandList> - -// Can't be named CALL as that conflicts with the opcode. -#define CALL_C(func) CallAction - -#define ADDDEBUGENTRY() AddDebugEntryAction - -#define BEGIN_RULE_TABLE void AutoTranslator::initTable() { -#define END_RULE_TABLE } - -#define BEGIN_RULES(__t) \ - { \ - auto& __rules = instr_rule_map_ \ - .emplace( \ - std::piecewise_construct, \ - std::forward_as_tuple(__t), \ - std::forward_as_tuple()) \ - .first->second; - -#define END_RULES } -#define GEN(s, actions...) \ - { \ - UNUSED constexpr char pattern[] = s; \ - using rule_actions = RuleActions; \ - auto gen = [](Environ* env, const Instruction* instr) { \ - rule_actions::eval(env, instr); \ - }; \ - __rules = addPattern(std::move(__rules), s, gen); \ - } - -// *********************************************************************** -// Definition of Auto Generation Table -// The table consisting of multiple rules, and the rules for the same LIR -// instruction are grouped by BEGIN_RULES(LIR instruction type) and -// END_RULES. -// GEN defines a rule for a certain operand pattern of the LIR instruction, -// and maps it to a list of actions: -// GEN(, action1, action2, ...) +// Tear down the frame. On x86, this executes 'leave' (mov rsp, rbp; pop rbp). +// On aarch64, this restores sp from fp and pops the frame record (fp + lr). // -// The operand pattern is defined by a string, and each character in the string -// correpsonds to an operand of the instruction. The character can be one -// of the following: -// * 'R' - general purpose register operand output -// * 'r' - general purpose register operand input -// * 'X' - floating-point register operand output -// * 'x' - floating-point register operand input -// * 'i' - immediate operand input -// * 'M' - memory stack operand output -// * 'm' - memory stack operand input -// Wildcards "?" and "*" can also be used in patterns, where "?" represents any -// one of the types listed above and "*" represents one or more above types. -// Please note that while "?" can appear anywhere in a pattern, "*" can only be -// used at the end of a pattern. -// The actions can be ASM and CALL_C, meaning generating an assembly instruction -// and call a user-defined function, respectively. The first argument of ASM -// action is the mnemonic of the instruction to be generated, and the following -// arguments are the operands to the instruction. Currently, we have four types -// of assembly instruction operands: -// * OP - either an immediate operand or register oeprand -// * STK - a memory stack location [RBP - ?] -// * LBL - a label to a basic block -// * MEM - a memory operand. The size of the memory operand will be set to the -// size of the LIR instruction operand specified by the first argument -// of MEM. -// The assembly instruction operands are constructed from one or more LIR -// instruction operands. To specify the LIR operands, we use indices -// of the pattern string. For example: -// GEN("Rri", ASM(mov, OP(0), MEM(0, 1, 2))) -// means generating a mov instruction, whose first operand is a -// register/immediate operand, constructed from the only output of the LIR -// instruction, and the second operand is memory operand, constructed from the -// register input and the immediate input of the LIR instruction. The size of -// the memory operand is set to the size of the output of the LIR instruction. -// *********************************************************************** +// No inputs. +void translateLeave(Environ* env) { + arch::Builder* as = env->as; #if defined(CINDER_X86_64) -// clang-format off -BEGIN_RULE_TABLE - -BEGIN_RULES(Instruction::kLea) - GEN("Rm", ASM(lea, OP(0), MEM(1))) -END_RULES - -BEGIN_RULES(Instruction::kCall) - GEN("Ri", ASM(call, OP(1)), ADDDEBUGENTRY()) - GEN("Rr", ASM(call, OP(1)), ADDDEBUGENTRY()) - GEN("i", ASM(call, OP(0)), ADDDEBUGENTRY()) - GEN("r", ASM(call, OP(0)), ADDDEBUGENTRY()) - GEN("m", ASM(call, STK(0)), ADDDEBUGENTRY()) -END_RULES - -BEGIN_RULES(Instruction::kMove) - GEN("Rr", ASM(mov, OP(0), OP(1))) - GEN("Ri", ASM(mov, OP(0), OP(1))) - GEN("Rm", ASM(mov, OP(0), MEM(1))) - GEN("Mr", ASM(mov, MEM(0), OP(1))) - GEN("Mi", ASM(mov, MEM(0), OP(1))) - GEN("Xx", ASM(movsd, OP(0), OP(1))) - GEN("Xm", ASM(movsd, OP(0), MEM(1))) - GEN("Mx", ASM(movsd, MEM(0), OP(1))) - GEN("Xr", ASM(movq, OP(0), OP(1))) - GEN("Rx", ASM(movq, OP(0), OP(1))) -END_RULES - -// Atomic move with relaxed ordering. -// On x86-64, relaxed loads/stores are plain mov. -// This corresponds to the C++/C memory_order_relaxed. -BEGIN_RULES(Instruction::kMoveRelaxed) - GEN("Rm", ASM(mov, OP(0), MEM(1))) - GEN("Mr", ASM(mov, MEM(0), OP(1))) - GEN("Mi", ASM(mov, MEM(0), OP(1))) -END_RULES - - -BEGIN_RULES(Instruction::kGuard) - GEN(ANY, CALL_C(TranslateGuard)); -END_RULES - -BEGIN_RULES(Instruction::kDeoptPatchpoint) - GEN(ANY, CALL_C(TranslateDeoptPatchpoint)); -END_RULES - -BEGIN_RULES(Instruction::kNegate) - GEN("r", ASM(neg, OP(0))) - GEN("Ri", ASM(mov, OP(0), ImmOperandNegate)) - GEN("Rr", ASM(mov, OP(0), OP(1)), ASM(neg, OP(0))) - GEN("Rm", ASM(mov, OP(0), STK(1)), ASM(neg, OP(0))) -END_RULES - -BEGIN_RULES(Instruction::kInvert) - GEN("Ri", ASM(mov, OP(0), ImmOperandInvert)) - GEN("Rr", ASM(mov, OP(0), OP(1)), ASM(not_, OP(0))) - GEN("Rm", ASM(mov, OP(0), STK(1)), ASM(not_, OP(0))) -END_RULES - -BEGIN_RULES(Instruction::kMovZX) - GEN("Rr", ASM(movzx, OP(0), OP(1))) - GEN("Rm", ASM(movzx, OP(0), STK(1))) -END_RULES - -BEGIN_RULES(Instruction::kMovSX) - GEN("Rr", ASM(movsx, OP(0), OP(1))) - GEN("Rm", ASM(movsx, OP(0), STK(1))) -END_RULES - -BEGIN_RULES(Instruction::kMovSXD) - GEN("Rr", ASM(movsxd, OP(0), OP(1))) - GEN("Rm", ASM(movsxd, OP(0), STK(1))) -END_RULES - -BEGIN_RULES(Instruction::kUnreachable) - GEN(ANY, ASM(ud2)) -END_RULES - -#define DEF_BINARY_OP_RULES(name, instr) \ - BEGIN_RULES(Instruction::name) \ - GEN("ri", ASM(instr, OP(0), OP(1))) \ - GEN("rr", ASM(instr, OP(0), OP(1))) \ - GEN("rm", ASM(instr, OP(0), STK(1))) \ - /* rewriteBinaryOpInstrs() makes it safe to write the output before reading - * all inputs without inputs_live_across being set for most binary ops; see - * postalloc.cpp for details. */ \ - GEN("Rri", ASM(mov, OP(0), OP(1)), ASM(instr, OP(0), OP(2))) \ - GEN("Rrr", ASM(mov, OP(0), OP(1)), ASM(instr, OP(0), OP(2))) \ - GEN("Rrm", ASM(mov, OP(0), OP(1)), ASM(instr, OP(0), STK(2))) \ - END_RULES - -DEF_BINARY_OP_RULES(kAdd, add) -DEF_BINARY_OP_RULES(kSub, sub) -DEF_BINARY_OP_RULES(kAnd, and_) -DEF_BINARY_OP_RULES(kOr, or_) -DEF_BINARY_OP_RULES(kXor, xor_) -DEF_BINARY_OP_RULES(kMul, imul) - -BEGIN_RULES(Instruction::kDiv) - GEN("rrr", ASM(idiv, OP(0), OP(1), OP(2)) ) - GEN("rrm", ASM(idiv, OP(0), OP(1), STK(2)) ) - GEN("rr", ASM(idiv, OP(0), OP(1)) ) - GEN("rm", ASM(idiv, OP(0), STK(1)) ) -END_RULES - -BEGIN_RULES(Instruction::kDivUn) - GEN("rrr", ASM(div, OP(0), OP(1), OP(2)) ) - GEN("rrm", ASM(div, OP(0), OP(1), STK(2)) ) - GEN("rr", ASM(div, OP(0), OP(1)) ) - GEN("rm", ASM(div, OP(0), STK(1)) ) -END_RULES - -#undef DEF_BINARY_OP_RULES - -BEGIN_RULES(Instruction::kFadd) - /* rewriteBinaryOpInstrs() makes it safe to write the output before reading - * all inputs without inputs_live_across being set for Fadd; see - * postalloc.cpp for details. */ - GEN("Xxx", ASM(movsd, OP(0), OP(1)), ASM(addsd, OP(0), OP(2))) - GEN("xx", ASM(addsd, OP(0), OP(1))) -END_RULES - -BEGIN_RULES(Instruction::kFsub) - GEN("Xxx", ASM(movsd, OP(0), OP(1)), ASM(subsd, OP(0), OP(2))) - GEN("xx", ASM(subsd, OP(0), OP(1))) -END_RULES - -BEGIN_RULES(Instruction::kFmul) - /* rewriteBinaryOpInstrs() makes it safe to write the output before reading - * all inputs without inputs_live_across being set for Fmul; see - * postalloc.cpp for details. */ - GEN("Xxx", ASM(movsd, OP(0), OP(1)), ASM(mulsd, OP(0), OP(2))) - GEN("xx", ASM(mulsd, OP(0), OP(1))) -END_RULES - -BEGIN_RULES(Instruction::kFdiv) - GEN("Xxx", ASM(movsd, OP(0), OP(1)), ASM(divsd, OP(0), OP(2))) - GEN("xx", ASM(divsd, OP(0), OP(1))) -END_RULES - -BEGIN_RULES(Instruction::kPush) - GEN("r", ASM(push, OP(0))) - GEN("m", ASM(push, STK(0))) - GEN("i", ASM(push, OP(0))) -END_RULES - -BEGIN_RULES(Instruction::kPop) - GEN("R", ASM(pop, OP(0))) - GEN("M", ASM(pop, STK(0))) -END_RULES - -BEGIN_RULES(Instruction::kCdq) - GEN("Rr", ASM(cdq, OP(0), OP(1))) -END_RULES - -BEGIN_RULES(Instruction::kCwd) - GEN("Rr", ASM(cwd, OP(0), OP(1))) -END_RULES - -BEGIN_RULES(Instruction::kCqo) - GEN("Rr", ASM(cqo, OP(0), OP(1))) -END_RULES - -BEGIN_RULES(Instruction::kExchange) - GEN("Rr", ASM(xchg, OP(0), OP(1))) - GEN("Xx", ASM(pxor, OP(0), OP(1)), - ASM(pxor, OP(1), OP(0)), - ASM(pxor, OP(0), OP(1))) -END_RULES - -BEGIN_RULES(Instruction::kCmp) - GEN("rr", ASM(cmp, OP(0), OP(1))) - GEN("ri", ASM(cmp, OP(0), OP(1))) - GEN("xx", ASM(comisd, OP(0), OP(1))) -END_RULES - -BEGIN_RULES(Instruction::kTest) - GEN("rr", ASM(test, OP(0), OP(1))) -END_RULES - -BEGIN_RULES(Instruction::kTest32) - GEN("rr", ASM(test, REG_OP(0, 32), REG_OP(1, 32))) -END_RULES - -BEGIN_RULES(Instruction::kBranch) - GEN("b", ASM(jmp, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchZ) - GEN("b", ASM(jz, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchNZ) - GEN("b", ASM(jnz, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchA) - GEN("b", ASM(ja, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchB) - GEN("b", ASM(jb, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchAE) - GEN("b", ASM(jae, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchBE) - GEN("b", ASM(jbe, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchG) - GEN("b", ASM(jg, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchL) - GEN("b", ASM(jl, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchGE) - GEN("b", ASM(jge, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchLE) - GEN("b", ASM(jle, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchC) - GEN("b", ASM(jc, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchNC) - GEN("b", ASM(jnc, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchO) - GEN("b", ASM(jo, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchNO) - GEN("b", ASM(jno, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchS) - GEN("b", ASM(js, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchNS) - GEN("b", ASM(jns, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchE) - GEN("b", ASM(je, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchNE) - GEN("b", ASM(jne, LBL(0))) -END_RULES - -#define DEF_COMPARE_OP_RULES(name, fpcomp) \ -BEGIN_RULES(Instruction::name) \ - GEN("Rrr", CALL_C(TranslateCompare)) \ - GEN("Rri", CALL_C(TranslateCompare)) \ - GEN("Rrm", CALL_C(TranslateCompare)) \ - if (fpcomp) { \ - GEN("Rxx", CALL_C(TranslateCompare)) \ - } \ -END_RULES - -DEF_COMPARE_OP_RULES(kEqual, true) -DEF_COMPARE_OP_RULES(kNotEqual, true) -DEF_COMPARE_OP_RULES(kGreaterThanUnsigned, true) -DEF_COMPARE_OP_RULES(kGreaterThanEqualUnsigned, true) -DEF_COMPARE_OP_RULES(kLessThanUnsigned, true) -DEF_COMPARE_OP_RULES(kLessThanEqualUnsigned, true) -DEF_COMPARE_OP_RULES(kGreaterThanSigned, false) -DEF_COMPARE_OP_RULES(kGreaterThanEqualSigned, false) -DEF_COMPARE_OP_RULES(kLessThanSigned, false) -DEF_COMPARE_OP_RULES(kLessThanEqualSigned, false) - -#undef DEF_COMPARE_OP_RULES - -BEGIN_RULES(Instruction::kInc) - GEN("r", ASM(inc, OP(0))) - GEN("m", ASM(inc, STK(0))) -END_RULES - -BEGIN_RULES(Instruction::kDec) - GEN("r", ASM(dec, OP(0))) - GEN("m", ASM(dec, STK(0))) -END_RULES - -BEGIN_RULES(Instruction::kBitTest) - GEN("ri", ASM(bt, OP(0), OP(1))) -END_RULES - -BEGIN_RULES(Instruction::kYieldInitial) - GEN(ANY, CALL_C(translateYieldInitial)) -END_RULES - -#if PY_VERSION_HEX < 0x030C0000 -BEGIN_RULES(Instruction::kYieldFrom) - GEN(ANY, CALL_C(translateYieldFrom)) -END_RULES + as->leave(); +#elif defined(CINDER_AARCH64) + as->mov(a64::sp, arch::fp); + as->ldp(arch::fp, arch::lr, a64::ptr_post(a64::sp, arch::kFrameRecordSize)); + env->sp_to_fp_delta = arch::kSpPositionUnknown; #else -// In 3.12+ YieldFrom is a pseudo-op which is YieldValue plus enough -// information to know which live value contains the target iterator. See -// emitStoreGenYieldPoint() for where this is captured. The target iterator is -// used for things like the result of reading gi_yieldfrom. -BEGIN_RULES(Instruction::kYieldFrom) - GEN(ANY, CALL_C(translateYieldValue)) -END_RULES + CINDER_UNSUPPORTED #endif +} -BEGIN_RULES(Instruction::kYieldFromSkipInitialSend) - GEN(ANY, CALL_C(translateYieldFrom)) -END_RULES - -BEGIN_RULES(Instruction::kYieldFromHandleStopAsyncIteration) - GEN(ANY, CALL_C(translateYieldFrom)) -END_RULES +// Return from a function. On x86, this is 'ret'. On aarch64, 'ret lr'. +// +// No inputs. +void translateRet(Environ* env) { + arch::Builder* as = env->as; -BEGIN_RULES(Instruction::kYieldValue) - GEN(ANY, CALL_C(translateYieldValue)) -END_RULES +#if defined(CINDER_X86_64) + as->ret(); +#elif defined(CINDER_AARCH64) + as->ret(arch::lr); +#else + CINDER_UNSUPPORTED +#endif +} -BEGIN_RULES(Instruction::kSelect) - GEN("Rrri", ASM(mov, OP(0), OP(3)), - ASM(test, OP(1), OP(1)), - ASM(cmovnz, OP(0), OP(2))) -END_RULES +void translateShift(Environ* env, const Instruction* instr) { + auto opcode = instr->opcode(); + auto in0_reg = getReg(instr, instr->getInput(0)); + auto in1 = instr->getInput(1); + auto out_reg = + (instr->getNumOutputs() > 0) ? getReg(instr, instr->output()) : in0_reg; + // Currently just a limitation of x86-64 register allocation. + JIT_CHECK( + kBuildArch != Arch::kX86_64 || in1->isImm(), + "Cannot emit non-immediate RHS for instruction '{}'", + *instr); -BEGIN_RULES(Instruction::kIntToBool) - GEN("Rr", CALL_C(translateIntToBool)) - GEN("Ri", CALL_C(translateIntToBool)) -END_RULES + if (instr->getNumOutputs() > 0 && kBuildArch != Arch::kAarch64) { + env->as->mov(out_reg, in0_reg); + } -END_RULE_TABLE -// clang-format on +#if defined(CINDER_X86_64) + asmjit::Imm shift = getImm(in1); + switch (opcode) { + case Opcode::kLShift: + env->as->shl(out_reg, shift); + return; + case Opcode::kRShift: + env->as->sar(out_reg, shift); + return; + case Opcode::kRShiftUn: + env->as->shr(out_reg, shift); + return; + default: + break; + } #elif defined(CINDER_AARCH64) + switch (opcode) { + case Opcode::kLShift: + if (in1->isReg()) { + env->as->lsl(out_reg, in0_reg, getReg(instr, in1)); + } else { + env->as->lsl(out_reg, in0_reg, getImm(in1)); + } + return; + case Opcode::kRShift: + if (in1->isReg()) { + env->as->asr(out_reg, in0_reg, getReg(instr, in1)); + } else { + env->as->asr(out_reg, in0_reg, getImm(in1)); + } + return; + case Opcode::kRShiftUn: + if (in1->isReg()) { + env->as->lsr(out_reg, in0_reg, getReg(instr, in1)); + } else { + env->as->lsr(out_reg, in0_reg, getImm(in1)); + } + return; + default: + break; + } +#else + JIT_ABORT("Unrecognized architecture for emitting shift instruction"); +#endif + JIT_ABORT("Unrecognized shift opcode '{}'", instr->opname()); +} +#if defined(CINDER_AARCH64) namespace { using AT = AutoTranslator; @@ -1754,7 +1592,7 @@ using AT = AutoTranslator; // We do not want to extend AT::getGp to support SP because we only want to // return SP in very specific circumstances (e.g., building an address relative // to SP). -arch::Gp getGpOrSP(const OperandBase* operand) { +arch::Gp getGpOrSP(const lir::Operand* operand) { if (operand->getPhyRegister() == SP) { return a64::sp; } else { @@ -1787,13 +1625,11 @@ void leaIndex( case 3: as->add(output, base, index, a64::lsl(3)); break; - default: { - // Use scratch register to avoid clobbering index when output and - // index are the same register. - as->mov(arch::reg_scratch_0, uint64_t{1} << multiplier); - as->madd(output, index, arch::reg_scratch_0, base); - break; - } + default: + JIT_ABORT( + "Unexpected multiplier {} in leaIndex - should have been lowered " + "by postgen rewrite", + multiplier); } } @@ -1802,7 +1638,6 @@ void leaIndex( void leaIndirect( arch::Builder* as, arch::Gp output, - arch::Gp scratch0, const MemoryIndirect* indirect) { auto base = getGpOrSP(indirect->getBaseRegOperand()); auto indexRegOperand = indirect->getIndexRegOperand(); @@ -1814,28 +1649,11 @@ void leaIndirect( output, base, AT::getGp(indexRegOperand), - indirect->getMultipiler()); + indirect->getMultiplier()); base = output; } - - if (offset > 0) { - if (arm::Utils::isAddSubImm(static_cast(offset))) { - as->add(output, base, offset); - } else { - as->mov(scratch0, offset); - as->add(output, base, scratch0); - } - } else if (offset < 0) { - if (arm::Utils::isAddSubImm(static_cast(-offset))) { - as->sub(output, base, -offset); - } else { - as->mov(scratch0, -offset); - as->sub(output, base, scratch0); - } - } else if (indexRegOperand == nullptr) { - as->mov(output, base); - } + arch::add_signed_immediate(as, output, base, offset); } // Resolve the memory address represented by a MemoryIndirect into an a64::Mem @@ -1844,18 +1662,25 @@ arch::Mem ptrIndirect( arch::Builder* as, arch::Gp scratch0, arch::Gp scratch1, - const MemoryIndirect* indirect) { + const MemoryIndirect* indirect, + DataType data_type) { auto base = getGpOrSP(indirect->getBaseRegOperand()); auto indexRegOperand = indirect->getIndexRegOperand(); auto offset = indirect->getOffset(); if (indexRegOperand != nullptr) { - leaIndex( - as, - scratch1, - base, - AT::getGp(indexRegOperand), - indirect->getMultipiler()); + auto index = AT::getGp(indexRegOperand); + auto multiplier = indirect->getMultiplier(); + + if (offset == 0) { + if (multiplier == 0) { + return a64::ptr(base, index); + } else if (multiplier == byteShift(data_type)) { + return a64::ptr(base, index, a64::lsl(multiplier)); + } + } + + leaIndex(as, scratch1, base, index, multiplier); base = scratch1; } @@ -1865,21 +1690,22 @@ arch::Mem ptrIndirect( void loadToReg( arch::Builder* as, - const OperandBase* output, + const lir::Operand* output, const arch::Mem& input) { if (output->isVecD()) { as->ldr(AT::getVecD(output), input); } else { - auto reg = AT::getGp(output); switch (output->dataType()) { - case OperandBase::k8bit: - as->ldrb(reg, input); + case lir::Operand::k8bit: + as->ldrb( + AT::getGp(DataType::k32bit, output->getPhyRegister().loc), input); break; - case OperandBase::k16bit: - as->ldrh(reg, input); + case lir::Operand::k16bit: + as->ldrh( + AT::getGp(DataType::k32bit, output->getPhyRegister().loc), input); break; default: - as->ldr(reg, input); + as->ldr(AT::getGp(output), input); break; } } @@ -1887,22 +1713,25 @@ void loadToReg( void storeFromReg( arch::Builder* as, - const OperandBase* input, + const lir::Operand* input, + const lir::Operand* output_operand, const arch::Mem& output) { if (input->isVecD()) { as->str(AT::getVecD(input), output); } else { - switch (input->dataType()) { - case OperandBase::k8bit: + switch (output_operand->dataType()) { + case lir::Operand::k8bit: as->strb( AT::getGp(DataType::k32bit, input->getPhyRegister().loc), output); break; - case OperandBase::k16bit: + case lir::Operand::k16bit: as->strh( AT::getGp(DataType::k32bit, input->getPhyRegister().loc), output); break; default: - as->str(AT::getGp(input), output); + as->str( + AT::getGp(output_operand->dataType(), input->getPhyRegister().loc), + output); break; } } @@ -1917,13 +1746,18 @@ void translateLea(Environ* env, const Instruction* instr) { JIT_CHECK(output->isReg(), "Expected output to be a register"); if (input->isStack()) { - as->add(AT::getGp(output), arch::fp, input->getStackSlot().loc); + arch::add_signed_immediate( + as, getGpOrSP(output), arch::fp, input->getStackSlot().loc); } else if (input->isMem()) { auto address = reinterpret_cast(input->getMemoryAddress()); - as->mov(AT::getGp(output), address); + as->mov(getGpOrSP(output), address); } else if (input->isInd()) { - leaIndirect( - as, AT::getGp(output), arch::reg_scratch_0, input->getMemoryIndirect()); + leaIndirect(as, getGpOrSP(output), input->getMemoryIndirect()); + } else if (input->isLabel()) { + asmjit::Label label = input->getDefine()->hasAsmLabel() + ? input->getDefine()->getAsmLabel() + : map_get(env->block_label_map, input->getBasicBlock()); + as->adr(getGpOrSP(output), label); } else { JIT_ABORT("Unsupported operand type for Lea: {}", input->type()); } @@ -1935,17 +1769,24 @@ void translateCall(Environ* env, const Instruction* instr) { auto output = instr->output(); auto input = instr->getInput(0); - if (input->isReg()) { + if (input->isImm()) { + // Use bl(imm) so asmjit can pick the final encoding at relocation time: + // direct bl if within ±128MB, or a branch to an out-of-line stub. + as->bl(static_cast(input->getConstant())); + } else if (input->isReg()) { as->blr(AT::getGp(input)); - } else if (input->isImm()) { - as->mov(arch::reg_scratch_br, input->getConstant()); - as->blr(arch::reg_scratch_br); } else if (input->isStack()) { - auto loc = input->getStackSlot().loc; as->ldr( - arch::reg_scratch_br, - arch::ptr_resolve(as, arch::fp, loc, arch::reg_scratch_0)); + arch::reg_scratch_br, getStackSlotPtr(env, input->getStackSlot().loc)); as->blr(arch::reg_scratch_br); + } else if (input->isImm()) { + as->mov(arch::reg_scratch_br, input->getConstant()); + as->blr(arch::reg_scratch_br); + } else if (input->isLabel()) { + asmjit::Label label = input->getDefine()->hasAsmLabel() + ? input->getDefine()->getAsmLabel() + : map_get(env->block_label_map, input->getBasicBlock()); + as->bl(label); } else { JIT_ABORT("Unsupported operand type for Call: {}", input->type()); } @@ -1956,11 +1797,11 @@ void translateCall(Environ* env, const Instruction* instr) { env->pending_debug_locs.emplace_back(label, instr->origin()); } - if (output->type() != OperandBase::kNone) { + if (output->type() != lir::Operand::kNone) { if (output->isVecD()) { as->mov(AT::getVecD(output), a64::d0); } else { - auto out_reg = AT::getGp(output); + auto out_reg = AT::getGpOutput(output); // Match the source register width to the destination register width. // aarch64 mov requires both operands to be the same size. if (out_reg.isGpW()) { @@ -1987,8 +1828,12 @@ void translateMove(Environ* env, const Instruction* instr) { auto scratch0 = arch::reg_scratch_0; auto scratch1 = arch::reg_scratch_1; - const OperandBase* output = instr->output(); - const OperandBase* input = instr->getInput(0); + const lir::Operand* output = instr->output(); + const lir::Operand* input = instr->getInput(0); + + if (instr->isMoveRelaxed()) { + checkMoveRelaxedOperandShape(instr); + } switch (output->type()) { case lir::OperandType::kReg: @@ -2005,36 +1850,36 @@ void translateMove(Environ* env, const Instruction* instr) { if (input->isVecD()) { as->fmov(AT::getGp(output), AT::getVecD(input)); } else { - as->mov(AT::getGp(output), AT::getGp(input)); + as->mov(AT::getGpWiden(output), AT::getGpWiden(input)); } } break; case lir::OperandType::kStack: { // Loading a value from the stack into a register. - auto ptr = arch::ptr_resolve( - as, arch::fp, input->getStackSlot().loc, arch::reg_scratch_0); if (output->isVecD()) { - as->ldr(AT::getVecD(output), ptr); - } else { - switch (output->dataType()) { - case OperandBase::k8bit: - as->ldrb(AT::getGpOutput(output), ptr); - break; - case OperandBase::k16bit: - as->ldrh(AT::getGpOutput(output), ptr); - break; - default: - as->ldr(AT::getGp(output), ptr); - break; - } + as->ldr( + AT::getVecD(output), + getStackSlotPtr(env, input->getStackSlot().loc)); + break; + } + // We use dst here rather than scratch because postalloc could + // have inserted use of the scratch register and we're about + // to clobber dst anyway. + auto dst = a64::x(output->getPhyRegister().loc); + auto ptr = getStackSlotPtr(env, input->getStackSlot().loc, dst); + switch (output->dataType()) { + case lir::Operand::k8bit: + as->ldrb(AT::getGpOutput(output), ptr); + break; + case lir::Operand::k16bit: + as->ldrh(AT::getGpOutput(output), ptr); + break; + default: + as->ldr(AT::getGp(output), ptr); + break; } break; } - case lir::OperandType::kMem: - // Loading a value from an absolute address into a register. - as->mov(arch::reg_scratch_0, input->getMemoryAddress()); - loadToReg(as, output, a64::ptr(arch::reg_scratch_0)); - break; case lir::OperandType::kInd: { // Loading a value from an address relative to another register into // a register. @@ -2042,51 +1887,69 @@ void translateMove(Environ* env, const Instruction* instr) { as, arch::reg_scratch_0, arch::reg_scratch_1, - input->getMemoryIndirect()); + input->getMemoryIndirect(), + output->dataType()); loadToReg(as, output, ptr); break; } - case lir::OperandType::kImm: + case lir::OperandType::kImm: { // Loading a constant immediate into a register. + auto constant = input->getConstant(); + if (output->isVecD()) { - as->fmov(AT::getVecD(output), input->getConstant()); + as->fmov(AT::getVecD(output), constant); + } else if (constant == 0) { + as->mov( + AT::getGpWiden(output), + AT::getGpWiden(output->dataType(), a64::xzr.id())); + } else if (input->dataType() == lir::Operand::kObject) { + // Pointer constant: use load_addr which relaxes to adr, adrp+add, + // or ldr from address table depending on displacement. + // load_addr emits adr which requires a 64-bit x register. + as->load_addr( + a64::x(output->getPhyRegister().loc), + static_cast(constant)); } else { - as->mov(AT::getGp(output), input->getConstant()); + as->mov(AT::getGpWiden(output), constant); } break; + } case lir::OperandType::kNone: case lir::OperandType::kVreg: + case lir::OperandType::kMem: case lir::OperandType::kLabel: JIT_ABORT( "Unsupported operand type for Move: Reg + {}", input->type()); } break; case lir::OperandType::kStack: { - auto ptr = arch::ptr_resolve( - as, arch::fp, output->getStackSlot().loc, arch::reg_scratch_0); - - if (input->isReg()) { - // Storing the value of a register to the stack. - storeFromReg(as, input, ptr); - } else if (input->isImm()) { - // Storing a constant immediate to the stack. - as->mov(scratch0, input->getConstant()); - as->str(scratch0, ptr); - } else { + if (!input->isReg()) { JIT_ABORT("Unsupported operand type for Move: Stk + {}", input->type()); } + // A store has no spare destination to build the address in, so it has to + // use a scratch -- but not the one holding the value it is about to + // write, or the address would overwrite it first. + auto scratch = input->getPhyRegister() == arch::reg_scratch_0_loc + ? arch::reg_scratch_1 + : arch::reg_scratch_0; + // Storing the value of a register to the stack + storeFromReg( + as, + input, + output, + getStackSlotPtr(env, output->getStackSlot().loc, scratch)); break; } case lir::OperandType::kMem: - as->mov(scratch0, reinterpret_cast(output->getMemoryAddress())); + as->load_addr(scratch0, output->getMemoryAddress()); if (input->isReg()) { // Storing the value of a register to an absolute address. if (input->isVecD()) { as->str(AT::getVecD(input), a64::ptr(scratch0)); } else { - as->str(AT::getGp(input), a64::ptr(scratch0)); + as->str(AT::getGpWiden(input), a64::ptr(scratch0)); } } else if (input->isImm()) { // Storing a constant immediate to an absolute address. @@ -2100,26 +1963,40 @@ void translateMove(Environ* env, const Instruction* instr) { if (input->isReg()) { // Storing the value of a register to an address relative to another // register. - auto ptr = - ptrIndirect(as, scratch0, scratch1, output->getMemoryIndirect()); + auto ptr = ptrIndirect( + as, + scratch0, + scratch1, + output->getMemoryIndirect(), + output->dataType()); - storeFromReg(as, input, ptr); + storeFromReg(as, input, output, ptr); } else if (input->isImm()) { // Storing a constant immediate to an address relative to another // register. - auto ptr = - ptrIndirect(as, scratch0, scratch1, output->getMemoryIndirect()); + auto ptr = ptrIndirect( + as, + scratch0, + scratch1, + output->getMemoryIndirect(), + output->dataType()); // Use the output's data type to determine the store width. switch (output->dataType()) { - case OperandBase::k8bit: + case lir::Operand::k8bit: as->mov(a64::w(scratch1.id()), input->getConstant()); as->strb(a64::w(scratch1.id()), ptr); break; - case OperandBase::k16bit: + case lir::Operand::k16bit: as->mov(a64::w(scratch1.id()), input->getConstant()); as->strh(a64::w(scratch1.id()), ptr); break; + case lir::Operand::k32bit: + // Use w register for 4-byte store to avoid overflowing + // tightly-packed fields. + as->mov(a64::w(scratch1.id()), input->getConstant()); + as->str(a64::w(scratch1.id()), ptr); + break; default: as->mov(scratch1, input->getConstant()); as->str(scratch1, ptr); @@ -2138,6 +2015,203 @@ void translateMove(Environ* env, const Instruction* instr) { } } +void translateLoad(Environ* env, const Instruction* instr) { + a64::Builder* as = env->as; + const lir::Operand* output = instr->output(); + const lir::Operand* input = instr->getInput(0); + + JIT_CHECK( + output->isReg(), + "Load output must be a register, got {}", + output->type()); + JIT_CHECK( + isMemoryMoveOperand(input), + "Load input must be memory (Stk/Mem/Ind), got {}", + input->type()); + + switch (input->type()) { + case lir::OperandType::kStack: { + if (output->isVecD()) { + as->ldr( + AT::getVecD(output), + getStackSlotPtr(env, input->getStackSlot().loc)); + } else { + auto dst = a64::x(output->getPhyRegister().loc); + auto ptr = getStackSlotPtr(env, input->getStackSlot().loc, dst); + switch (output->dataType()) { + case lir::Operand::k8bit: + as->ldrb(AT::getGpOutput(output), ptr); + break; + case lir::Operand::k16bit: + as->ldrh(AT::getGpOutput(output), ptr); + break; + default: + as->ldr(AT::getGp(output), ptr); + break; + } + } + break; + } + case lir::OperandType::kInd: { + auto ptr = ptrIndirect( + as, + arch::reg_scratch_0, + arch::reg_scratch_1, + input->getMemoryIndirect(), + output->dataType()); + loadToReg(as, output, ptr); + break; + } + case lir::OperandType::kMem: { + auto scratch0 = arch::reg_scratch_0; + as->load_addr(scratch0, input->getMemoryAddress()); + if (output->isVecD()) { + as->ldr(AT::getVecD(output), a64::ptr(scratch0)); + } else { + switch (output->dataType()) { + case lir::Operand::k8bit: + as->ldrb(AT::getGpOutput(output), a64::ptr(scratch0)); + break; + case lir::Operand::k16bit: + as->ldrh(AT::getGpOutput(output), a64::ptr(scratch0)); + break; + default: + as->ldr(AT::getGp(output), a64::ptr(scratch0)); + break; + } + } + break; + } + default: + JIT_ABORT("Unsupported operand type for Load: Reg + {}", input->type()); + } +} + +void translateStore(Environ* env, const Instruction* instr) { + a64::Builder* as = env->as; + auto scratch0 = arch::reg_scratch_0; + auto scratch1 = arch::reg_scratch_1; + + const lir::Operand* output = instr->output(); + const lir::Operand* input = instr->getInput(0); + + JIT_CHECK( + isMemoryMoveOperand(output), + "Store output must be memory (Stk/Mem/Ind), got {}", + output->type()); + JIT_CHECK( + input->isReg() || input->isImm(), + "Store input must be Reg or Imm, got {}", + input->type()); + + switch (output->type()) { + case lir::OperandType::kStack: { + if (!input->isReg()) { + as->mov(scratch1, input->getConstant()); + auto ptr = getStackSlotPtr(env, output->getStackSlot().loc, scratch1); + switch (output->dataType()) { + case lir::Operand::k8bit: + as->strb(a64::w(scratch1.id()), ptr); + break; + case lir::Operand::k16bit: + as->strh(a64::w(scratch1.id()), ptr); + break; + case lir::Operand::k32bit: + as->str(a64::w(scratch1.id()), ptr); + break; + default: + as->str(scratch1, ptr); + break; + } + } else { + auto scratch = input->getPhyRegister() == arch::reg_scratch_0_loc + ? arch::reg_scratch_1 + : arch::reg_scratch_0; + storeFromReg( + as, + input, + output, + getStackSlotPtr(env, output->getStackSlot().loc, scratch)); + } + break; + } + case lir::OperandType::kMem: { + as->load_addr(scratch0, output->getMemoryAddress()); + if (input->isReg()) { + if (input->isVecD()) { + as->str(AT::getVecD(input), a64::ptr(scratch0)); + } else { + as->str(AT::getGpWiden(input), a64::ptr(scratch0)); + } + } else { + as->mov(scratch1, input->getConstant()); + as->str(scratch1, a64::ptr(scratch0)); + } + break; + } + case lir::OperandType::kInd: { + if (input->isReg()) { + auto ptr = ptrIndirect( + as, + scratch0, + scratch1, + output->getMemoryIndirect(), + output->dataType()); + storeFromReg(as, input, output, ptr); + } else { + auto ptr = ptrIndirect( + as, + scratch0, + scratch1, + output->getMemoryIndirect(), + output->dataType()); + switch (output->dataType()) { + case lir::Operand::k8bit: + as->mov(a64::w(scratch1.id()), input->getConstant()); + as->strb(a64::w(scratch1.id()), ptr); + break; + case lir::Operand::k16bit: + as->mov(a64::w(scratch1.id()), input->getConstant()); + as->strh(a64::w(scratch1.id()), ptr); + break; + case lir::Operand::k32bit: + as->mov(a64::w(scratch1.id()), input->getConstant()); + as->str(a64::w(scratch1.id()), ptr); + break; + default: + as->mov(scratch1, input->getConstant()); + as->str(scratch1, ptr); + break; + } + } + break; + } + default: + JIT_ABORT( + "Unsupported output operand type for Store: {}", output->type()); + } +} + +void translateMovConstPool(Environ* env, const Instruction* instr) { + a64::Builder* as = env->as; + auto output = instr->output(); + auto input = instr->getInput(0); + uint64_t value = static_cast(input->getConstant()); + + // Look up or create constant pool entry for this value. + asmjit::Label label; + auto it = env->const_pool_labels.find(value); + if (it == env->const_pool_labels.end()) { + label = as->newLabel(); + env->const_pool_labels[value] = label; + } else { + label = it->second; + } + + // Load from constant pool via PC-relative ldr. + as->ldr(AT::getGpWiden(output), a64::ptr(label)); +} + template < typename EmitExt8Fn, typename EmitExt16Fn, @@ -2154,11 +2228,11 @@ void translateMovExtOp( a64::Builder* as = env->as; auto output = AT::getGpOutput(instr->output()); - const OperandBase* input = instr->getInput(0); + const lir::Operand* input = instr->getInput(0); int input_size = input->sizeInBits(); if (input->isReg()) { - auto input_reg = AT::getGp(input); + auto input_reg = AT::getGp(DataType::k32bit, input->getPhyRegister().loc); switch (input_size) { case 8: @@ -2168,34 +2242,31 @@ void translateMovExtOp( emit_ext16(as, output, input_reg); break; case 32: - as->mov(a64::w(output.id()), a64::w(input_reg.id())); + as->mov(a64::w(output.id()), input_reg); break; default: JIT_ABORT("Unsupported input size for {}: {}", opname, input_size); } } else if (input->isStack()) { auto loc = input->getStackSlot().loc; + // Address goes in the destination, not the shared scratch; see the kStack + // load in translateMove for why. Each of these emits a single load, so the + // address is consumed before the destination is written. + auto dst = a64::x(output.id()); switch (input_size) { case 8: emit_load8( - as, - output, - arch::ptr_resolve( - as, arch::fp, loc, arch::reg_scratch_0, arch::AccessSize::k8)); + as, output, getStackSlotPtr(env, loc, dst, arch::AccessSize::k8)); break; case 16: emit_load16( - as, - output, - arch::ptr_resolve( - as, arch::fp, loc, arch::reg_scratch_0, arch::AccessSize::k16)); + as, output, getStackSlotPtr(env, loc, dst, arch::AccessSize::k16)); break; case 32: as->ldr( a64::w(output.id()), - arch::ptr_resolve( - as, arch::fp, loc, arch::reg_scratch_0, arch::AccessSize::k32)); + getStackSlotPtr(env, loc, dst, arch::AccessSize::k32)); break; default: JIT_ABORT("Unsupported input size for {}: {}", opname, input_size); @@ -2205,51 +2276,106 @@ void translateMovExtOp( } } -void translateMovZX(Environ* env, const Instruction* instr) { +void translateZext(Environ* env, const Instruction* instr) { + // ARM64 uxtb/uxth/ldrb/ldrh only accept W-register destinations. + // Writing to W implicitly zeros the upper 32 bits of the X register, + // so this correctly zero-extends to 64 bits even for k64bit outputs. translateMovExtOp( env, instr, - "MovZX", - [](a64::Builder* as, auto... args) { as->uxtb(args...); }, - [](a64::Builder* as, auto... args) { as->uxth(args...); }, - [](a64::Builder* as, auto... args) { as->ldrb(args...); }, - [](a64::Builder* as, auto... args) { as->ldrh(args...); }); + "Zext", + [](a64::Builder* as, auto output, auto input) { + as->uxtb(a64::w(output.id()), input); + }, + [](a64::Builder* as, auto output, auto input) { + as->uxth(a64::w(output.id()), input); + }, + [](a64::Builder* as, auto output, auto mem) { + as->ldrb(a64::w(output.id()), mem); + }, + [](a64::Builder* as, auto output, auto mem) { + as->ldrh(a64::w(output.id()), mem); + }); } -void translateMovSX(Environ* env, const Instruction* instr) { +void translateSext(Environ* env, const Instruction* instr) { + // The shared helper's 32-bit path only zero-extends, so sign-extending from + // 32 bits needs sxtw/ldrsw here. + const lir::Operand* input = instr->getInput(0); + if (input->sizeInBits() == 32) { + a64::Builder* as = env->as; + + JIT_THROW_IF( + instr->output()->sizeInBits() != 64, + "Sign-extend from 32-bits should always go to 64-bits, got '{}' " + "instead", + *instr); + auto output = AT::getGpOutput(instr->output()); + + if (input->isReg()) { + as->sxtw(output, asmjit::a64::w(input->getPhyRegister().loc)); + } else if (input->isStack()) { + auto loc = input->getStackSlot().loc; + as->ldrsw( + output, + arch::ptr_resolve( + as, arch::fp, loc, arch::reg_scratch_0, arch::AccessSize::k32)); + } else { + JIT_THROW("Unsupported operand type for '{}'", *instr); + } + return; + } + translateMovExtOp( env, instr, - "MovSX", + "Sext", [](a64::Builder* as, auto... args) { as->sxtb(args...); }, [](a64::Builder* as, auto... args) { as->sxth(args...); }, [](a64::Builder* as, auto... args) { as->ldrsb(args...); }, [](a64::Builder* as, auto... args) { as->ldrsh(args...); }); } -void translateMovSXD(Environ* env, const Instruction* instr) { +void translateUnreachable(Environ* env, const Instruction* instr) { + a64::Builder* as = env->as; + + as->udf(0); +} + +void translateNegate(Environ* env, const Instruction* instr) { a64::Builder* as = env->as; - auto output = AT::getGpOutput(instr->output()); - const OperandBase* input = instr->getInput(0); + const lir::Operand* output = + instr->getNumOutputs() > 0 ? instr->output() : instr->getInput(0); + const lir::Operand* opnd0 = instr->getInput(0); - if (input->isReg()) { - auto input_reg = AT::getGp(input); - as->sxtw(output, input_reg); - } else if (input->isStack()) { - auto loc = input->getStackSlot().loc; - auto ptr = arch::ptr_resolve( - as, arch::fp, loc, arch::reg_scratch_0, arch::AccessSize::k32); - as->ldrsw(output, ptr); + JIT_CHECK(output->isReg(), "Expected output to be a register"); + + auto output_reg = AT::getGpOutput(output); + + if (opnd0->isReg()) { + as->neg(output_reg, AT::getGpWiden(opnd0)); } else { - JIT_ABORT("Unsupported operand type for MovSXD: {}", input->type()); + JIT_ABORT("Unsupported operand type for Negate: {}", opnd0->type()); } } -void translateUnreachable(Environ* env, const Instruction* instr) { +void translateInvert(Environ* env, const Instruction* instr) { a64::Builder* as = env->as; - as->udf(0); + const lir::Operand* output = + instr->getNumOutputs() > 0 ? instr->output() : instr->getInput(0); + const lir::Operand* opnd0 = instr->getInput(0); + + JIT_CHECK(output->isReg(), "Expected output to be a register"); + + auto output_reg = AT::getGpOutput(output); + + if (opnd0->isReg()) { + as->mvn(output_reg, AT::getGpWiden(opnd0)); + } else { + JIT_ABORT("Unsupported operand type for Invert: {}", opnd0->type()); + } } template @@ -2260,16 +2386,16 @@ void translateAddSubOp( EmitFn emit) { a64::Builder* as = env->as; - const OperandBase* output = + const lir::Operand* output = instr->getNumOutputs() > 0 ? instr->output() : instr->getInput(0); - const OperandBase* opnd0 = instr->getInput(0); - const OperandBase* opnd1 = instr->getInput(1); + const lir::Operand* opnd0 = instr->getInput(0); + const lir::Operand* opnd1 = instr->getInput(1); JIT_CHECK(output->isReg(), "Expected output to be a register"); JIT_CHECK(opnd0->isReg(), "Expected opnd0 to be a register"); - auto output_reg = AT::getGp(output); - auto opnd0_reg = AT::getGp(opnd0); + auto output_reg = AT::getGpOutput(output); + auto opnd0_reg = AT::getGpWiden(opnd0); if (opnd1->isImm()) { uint64_t constant = opnd1->getConstant(); @@ -2277,12 +2403,7 @@ void translateAddSubOp( emit(as, output_reg, opnd0_reg, constant); } else if (opnd1->isReg()) { - emit(as, output_reg, opnd0_reg, AT::getGp(opnd1)); - } else if (opnd1->isStack()) { - auto loc = opnd1->getStackSlot().loc; - auto ptr = arch::ptr_resolve(as, arch::fp, loc, arch::reg_scratch_0); - as->ldr(arch::reg_scratch_0, ptr); - emit(as, output_reg, opnd0_reg, arch::reg_scratch_0); + emit(as, output_reg, opnd0_reg, AT::getGpWiden(opnd1)); } else { JIT_ABORT("Unsupported operand type for {}: {}", opname, opnd1->type()); } @@ -2308,16 +2429,16 @@ void translateLogicalOp( EmitFn emit) { a64::Builder* as = env->as; - const OperandBase* output = + const lir::Operand* output = instr->getNumOutputs() > 0 ? instr->output() : instr->getInput(0); - const OperandBase* opnd0 = instr->getInput(0); - const OperandBase* opnd1 = instr->getInput(1); + const lir::Operand* opnd0 = instr->getInput(0); + const lir::Operand* opnd1 = instr->getInput(1); JIT_CHECK(output->isReg(), "Expected output to be a register"); JIT_CHECK(opnd0->isReg(), "Expected opnd0 to be a register"); - auto output_reg = AT::getGp(output); - auto opnd0_reg = AT::getGp(opnd0); + auto output_reg = AT::getGpWiden(output); + auto opnd0_reg = AT::getGpWiden(opnd0); if (opnd1->isImm()) { uint64_t constant = opnd1->getConstant(); @@ -2326,12 +2447,7 @@ void translateLogicalOp( emit(as, output_reg, opnd0_reg, constant); } else if (opnd1->isReg()) { - emit(as, output_reg, opnd0_reg, AT::getGp(opnd1)); - } else if (opnd1->isStack()) { - auto loc = opnd1->getStackSlot().loc; - auto ptr = arch::ptr_resolve(as, arch::fp, loc, arch::reg_scratch_0); - as->ldr(arch::reg_scratch_0, ptr); - emit(as, output_reg, opnd0_reg, arch::reg_scratch_0); + emit(as, output_reg, opnd0_reg, AT::getGpWiden(opnd1)); } else { JIT_ABORT("Unsupported operand type for {}: {}", opname, opnd1->type()); } @@ -2358,32 +2474,42 @@ void translateXor(Environ* env, const Instruction* instr) { void translateMul(Environ* env, const Instruction* instr) { a64::Builder* as = env->as; - const OperandBase* output = + const lir::Operand* output = instr->getNumOutputs() > 0 ? instr->output() : instr->getInput(0); - const OperandBase* opnd0 = instr->getInput(0); - const OperandBase* opnd1 = instr->getInput(1); + const lir::Operand* opnd0 = instr->getInput(0); + const lir::Operand* opnd1 = instr->getInput(1); JIT_CHECK(output->isReg(), "Expected output to be a register"); JIT_CHECK(opnd0->isReg(), "Expected opnd0 to be a register"); - auto output_reg = AT::getGp(output); - auto opnd0_reg = AT::getGp(opnd0); + auto output_reg = AT::getGpWiden(output); + auto opnd0_reg = AT::getGpWiden(opnd0); - if (opnd1->isImm()) { - as->mov(arch::reg_scratch_0, opnd1->getConstant()); - as->mul(output_reg, opnd0_reg, arch::reg_scratch_0); - } else if (opnd1->isReg()) { - as->mul(output_reg, opnd0_reg, AT::getGp(opnd1)); - } else if (opnd1->isStack()) { - auto loc = opnd1->getStackSlot().loc; - auto ptr = arch::ptr_resolve(as, arch::fp, loc, arch::reg_scratch_0); - as->ldr(arch::reg_scratch_0, ptr); - as->mul(output_reg, opnd0_reg, arch::reg_scratch_0); + if (opnd1->isReg()) { + as->mul(output_reg, opnd0_reg, AT::getGpWiden(opnd1)); } else { JIT_ABORT("Unsupported operand type for Mul: {}", opnd1->type()); } } +void translateMulAdd(Environ* env, const Instruction* instr) { + a64::Builder* as = env->as; + + auto output = instr->output(); + auto opnd0 = instr->getInput(0); + auto opnd1 = instr->getInput(1); + auto opnd2 = instr->getInput(2); + + JIT_CHECK(output->isReg(), "Expected output to be a register"); + JIT_CHECK(opnd0->isReg(), "Expected opnd0 to be a register"); + JIT_CHECK(opnd1->isReg(), "Expected opnd1 to be a register"); + JIT_CHECK(opnd2->isReg(), "Expected opnd2 to be a register"); + + // madd Rd, Rn, Rm, Ra => Rd = Ra + Rn * Rm + as->madd( + AT::getGp(output), AT::getGp(opnd0), AT::getGp(opnd1), AT::getGp(opnd2)); +} + template void translateDivOp( Environ* env, @@ -2392,24 +2518,31 @@ void translateDivOp( EmitFn emit) { a64::Builder* as = env->as; - const OperandBase* output = + const lir::Operand* output = instr->getNumOutputs() > 0 ? instr->output() : instr->getInput(0); - const OperandBase* opnd0 = instr->getInput(0); - const OperandBase* opnd1 = instr->getInput(1); + + // Division instructions may have an extra leading Imm{0} input (used by x86 + // for the high half of the dividend). Skip it on AArch64. + size_t base = 0; + if (instr->getNumInputs() == 3 && instr->getInput(0)->isImm()) { + base = 1; + } + const lir::Operand* opnd0 = instr->getInput(base); + const lir::Operand* opnd1 = instr->getInput(base + 1); JIT_CHECK(output->isReg(), "Expected output to be a register"); JIT_CHECK(opnd0->isReg(), "Expected opnd0 to be a register"); - auto output_reg = AT::getGp(output); - auto opnd0_reg = AT::getGp(opnd0); + // Use getGpOutput to get the correct register width. sdiv/udiv require all + // operands to be the same width. getGpOutput returns w(reg) for k32bit and + // x(reg) for k64bit, matching the hardware instruction requirements. + // (getGpWiden would return x(reg) for k32bit, causing sdiv to interpret + // zero-extended 32-bit values as 64-bit, giving wrong results for negatives.) + auto output_reg = AT::getGpOutput(output); + auto opnd0_reg = AT::getGpOutput(opnd0); if (opnd1->isReg()) { - emit(as, output_reg, opnd0_reg, AT::getGp(opnd1)); - } else if (opnd1->isStack()) { - auto loc = opnd1->getStackSlot().loc; - auto ptr = arch::ptr_resolve(as, arch::fp, loc, arch::reg_scratch_0); - as->ldr(arch::reg_scratch_0, ptr); - emit(as, output_reg, opnd0_reg, arch::reg_scratch_0); + emit(as, output_reg, opnd0_reg, AT::getGpOutput(opnd1)); } else { JIT_ABORT("Unsupported operand type for {}: {}", opname, opnd1->type()); } @@ -2430,37 +2563,41 @@ void translateDivUn(Environ* env, const Instruction* instr) { void translatePush(Environ* env, const Instruction* instr) { a64::Builder* as = env->as; - const OperandBase* operand = instr->getInput(0); + const lir::Operand* operand = instr->getInput(0); - if (operand->isImm()) { - as->mov(arch::reg_scratch_0, operand->getConstant()); - as->str(arch::reg_scratch_0, a64::ptr_pre(a64::sp, -16)); - } else if (operand->isReg()) { - auto reg = AT::getGp(operand); + if (operand->isReg()) { + auto reg = AT::getGpWiden(operand); as->str(reg, a64::ptr_pre(a64::sp, -16)); } else if (operand->isStack()) { - auto loc = operand->getStackSlot().loc; - auto ptr = arch::ptr_resolve(as, arch::fp, loc, arch::reg_scratch_1); + // Resolve the source slot before SP moves. + auto ptr = + getStackSlotPtr(env, operand->getStackSlot().loc, arch::reg_scratch_1); as->ldr(arch::reg_scratch_0, ptr); as->str(arch::reg_scratch_0, a64::ptr_pre(a64::sp, -16)); } else { JIT_ABORT("Unsupported operand type for push: {}", operand->type()); } + + env->adjustSp(16); } void translatePop(Environ* env, const Instruction* instr) { a64::Builder* as = env->as; - const OperandBase* operand = instr->output(); + const lir::Operand* operand = instr->output(); + + // SP is released by the load below, so the destination slot has to be + // resolved against the post-pop position. + env->adjustSp(-16); if (operand->isReg()) { - auto reg = AT::getGp(operand); + auto reg = AT::getGpWiden(operand); as->ldr(reg, a64::ptr_post(a64::sp, 16)); } else if (operand->isStack()) { - auto loc = operand->getStackSlot().loc; - auto ptr = arch::ptr_resolve(as, arch::fp, loc, arch::reg_scratch_1); as->ldr(arch::reg_scratch_0, a64::ptr_post(a64::sp, 16)); - as->str(arch::reg_scratch_0, ptr); + as->str( + arch::reg_scratch_0, + getStackSlotPtr(env, operand->getStackSlot().loc, arch::reg_scratch_1)); } else { JIT_ABORT("Unsupported operand type for pop: {}", operand->type()); } @@ -2469,8 +2606,8 @@ void translatePop(Environ* env, const Instruction* instr) { void translateExchange(Environ* env, const Instruction* instr) { a64::Builder* as = env->as; - const OperandBase* opnd0 = instr->output(); - const OperandBase* opnd1 = instr->getInput(0); + const lir::Operand* opnd0 = instr->output(); + const lir::Operand* opnd1 = instr->getInput(0); JIT_CHECK(opnd0->isReg(), "Expected opnd0 to be a register"); JIT_CHECK(opnd1->isReg(), "Expected opnd1 to be a register"); @@ -2479,13 +2616,13 @@ void translateExchange(Environ* env, const Instruction* instr) { auto vec0 = AT::getVecD(opnd0); auto vec1 = AT::getVecD(opnd1); - as->eor(vec0.v16(), vec0.v16(), vec1.v16()); - as->eor(vec1.v16(), vec1.v16(), vec0.v16()); - as->eor(vec0.v16(), vec0.v16(), vec1.v16()); + as->eor(vec0, vec0, vec1); + as->eor(vec1, vec1, vec0); + as->eor(vec0, vec0, vec1); } else { - auto reg0 = AT::getGp(opnd0); - auto reg1 = AT::getGp(opnd1); - auto scratch = arch::reg_scratch_0; + auto reg0 = AT::getGpWiden(opnd0); + auto reg1 = AT::getGpWiden(opnd1); + auto scratch = AT::getGpWiden(opnd0->dataType(), arch::reg_scratch_0.id()); as->mov(scratch, reg0); as->mov(reg0, reg1); @@ -2496,8 +2633,8 @@ void translateExchange(Environ* env, const Instruction* instr) { void translateCmp(Environ* env, const Instruction* instr) { a64::Builder* as = env->as; - const OperandBase* inp0 = instr->getInput(0); - const OperandBase* inp1 = instr->getInput(1); + const lir::Operand* inp0 = instr->getInput(0); + const lir::Operand* inp1 = instr->getInput(1); JIT_CHECK(inp0->isReg(), "Expected first input to be a register"); @@ -2505,17 +2642,11 @@ void translateCmp(Environ* env, const Instruction* instr) { if (inp0->isVecD() && inp1->isVecD()) { as->fcmp(AT::getVecD(inp0), AT::getVecD(inp1)); } else { - as->cmp(AT::getGp(inp0), AT::getGp(inp1)); + as->cmp(AT::getGpWiden(inp0), AT::getGpWiden(inp1)); } } else if (inp1->isImm()) { auto constant = inp1->getConstant(); - - if (arm::Utils::isAddSubImm(constant)) { - as->cmp(AT::getGp(inp0), constant); - } else { - as->mov(arch::reg_scratch_0, constant); - as->cmp(AT::getGp(inp0), arch::reg_scratch_0); - } + arch::cmp_immediate(as, AT::getGpWiden(inp0), constant); } else { JIT_ABORT( "Unsupported operand types for cmp: {} {}", inp0->type(), inp1->type()); @@ -2535,13 +2666,7 @@ void translateIncDecOp( if (opnd->isReg()) { // We have to do adds/subs here, because implicitly our LIR relies on the // Inc/Dec instructions setting flags. - emit(as, AT::getGp(opnd), AT::getGp(opnd), 1); - } else if (opnd->isStack()) { - auto loc = opnd->getStackSlot().loc; - auto ptr = arch::ptr_resolve(as, arch::fp, loc, arch::reg_scratch_1); - as->ldr(arch::reg_scratch_0, ptr); - emit(as, arch::reg_scratch_0, arch::reg_scratch_0, 1); - as->str(arch::reg_scratch_0, ptr); + emit(as, AT::getGpWiden(opnd), AT::getGpWiden(opnd), 1); } else { JIT_ABORT("Unsupported operand type for {}: {}", opname, opnd->dataType()); } @@ -2559,18 +2684,18 @@ void translateDec(Environ* env, const Instruction* instr) { }); } -void translateBitTest(Environ* env, const Instruction* instr) { +void translateBranchBit(Environ* env, const Instruction* instr, bool is_set) { a64::Builder* as = env->as; - auto test_reg = AT::getGp(instr->getInput(0)); + auto test_reg = AT::getGpWiden(instr->getInput(0)); auto bit_pos = instr->getInput(1)->getConstant(); + auto label = getLabel(env, instr->getInput(2)); - uint64_t mask = 1ULL << bit_pos; - JIT_CHECK( - arm::Utils::isLogicalImm(mask, 64), - "All single bits should be able to be tested"); - - as->tst(test_reg, mask); + if (is_set) { + as->tbnz(test_reg, bit_pos, label); + } else { + as->tbz(test_reg, bit_pos, label); + } } void translateTst(Environ* env, const Instruction* instr) { @@ -2584,9 +2709,9 @@ void translateTst(Environ* env, const Instruction* instr) { // 32-bit register using LSL so that TST sets the N and Z flags correctly for // the sub-register width. int shift = 0; - if (data_type == jit::lir::OperandBase::k8bit) { + if (data_type == jit::lir::Operand::k8bit) { shift = 24; - } else if (data_type == jit::lir::OperandBase::k16bit) { + } else if (data_type == jit::lir::Operand::k16bit) { shift = 16; } @@ -2608,8 +2733,8 @@ void translateSelect(Environ* env, const Instruction* instr) { auto condition_op = instr->getInput(0); arch::Gp condition_reg; switch (condition_op->dataType()) { - case jit::lir::OperandBase::k8bit: - case jit::lir::OperandBase::k16bit: + case jit::lir::Operand::k8bit: + case jit::lir::Operand::k16bit: condition_reg = AT::getGp(DataType::k32bit, condition_op->getPhyRegister().loc); as->and_( @@ -2621,367 +2746,946 @@ void translateSelect(Environ* env, const Instruction* instr) { condition_reg = AT::getGp(condition_op); break; } - auto true_val_reg = AT::getGp(instr->getInput(1)); - auto false_val = instr->getInput(2)->getConstant(); + auto true_val_reg = AT::getGpWiden(instr->getInput(1)); + auto false_val_reg = AT::getGpWiden(instr->getInput(2)); - as->mov(arch::reg_scratch_0, false_val); as->cmp(condition_reg, 0); - as->csel(output, true_val_reg, arch::reg_scratch_0, a64::CondCode::kNE); + as->csel(output, true_val_reg, false_val_reg, a64::CondCode::kNE); } } // namespace -// clang-format off -BEGIN_RULE_TABLE - -BEGIN_RULES(Instruction::kLea) - GEN("Rm", CALL_C(translateLea)) -END_RULES - -BEGIN_RULES(Instruction::kCall) - GEN("Ri", CALL_C(translateCall)) - GEN("Rr", CALL_C(translateCall)) - GEN("i", CALL_C(translateCall)) - GEN("r", CALL_C(translateCall)) - GEN("m", CALL_C(translateCall)) -END_RULES - -BEGIN_RULES(Instruction::kMove) - GEN("Rr", ASM(mov, OP(0), OP(1))) - GEN("Ri", CALL_C(translateMove)) - GEN("Rm", CALL_C(translateMove)) - GEN("Mr", CALL_C(translateMove)) - GEN("Mi", CALL_C(translateMove)) - GEN("Xx", ASM(fmov, OP(0), OP(1))) - GEN("Xm", CALL_C(translateMove)) - GEN("Mx", CALL_C(translateMove)) - GEN("Xr", ASM(fmov, OP(0), OP(1))) - GEN("Rx", ASM(fmov, OP(0), OP(1))) -END_RULES - -BEGIN_RULES(Instruction::kMoveRelaxed) - GEN("Rm", CALL_C(translateMove)) - GEN("Mr", CALL_C(translateMove)) - GEN("Mi", CALL_C(translateMove)) -END_RULES - -BEGIN_RULES(Instruction::kGuard) - GEN(ANY, CALL_C(TranslateGuard)) -END_RULES - -BEGIN_RULES(Instruction::kDeoptPatchpoint) - GEN(ANY, CALL_C(TranslateDeoptPatchpoint)) -END_RULES - -BEGIN_RULES(Instruction::kNegate) - GEN("r", ASM(neg, OP(0), OP(0))) - GEN("Ri", ASM(mov, OP(0), ImmOperandNegate)) - GEN("Rr", ASM(neg, OP(0), OP(1))) - GEN("Rm", ASM(ldr, OP(0), STK(1)), ASM(neg, OP(0), OP(0))) -END_RULES - -BEGIN_RULES(Instruction::kInvert) - GEN("Ri", ASM(mov, OP(0), ImmOperandInvert)) - GEN("Rr", ASM(mvn, OP(0), OP(1))) - GEN("Rm", ASM(ldr, OP(0), STK(1)), ASM(mvn, OP(0), OP(0))) -END_RULES - -BEGIN_RULES(Instruction::kMovZX) - GEN("Rr", CALL_C(translateMovZX)) - GEN("Rm", CALL_C(translateMovZX)) -END_RULES - -BEGIN_RULES(Instruction::kMovSX) - GEN("Rr", CALL_C(translateMovSX)) - GEN("Rm", CALL_C(translateMovSX)) -END_RULES - -BEGIN_RULES(Instruction::kMovSXD) - GEN("Rr", CALL_C(translateMovSXD)) - GEN("Rm", CALL_C(translateMovSXD)) -END_RULES - -BEGIN_RULES(Instruction::kUnreachable) - GEN(ANY, CALL_C(translateUnreachable)) -END_RULES - -BEGIN_RULES(Instruction::kAdd) - GEN("ri", CALL_C(translateAdd)) - GEN("rr", CALL_C(translateAdd)) - GEN("rm", CALL_C(translateAdd)) - GEN("Rri", CALL_C(translateAdd)) - GEN("Rrr", CALL_C(translateAdd)) - GEN("Rrm", CALL_C(translateAdd)) -END_RULES - -BEGIN_RULES(Instruction::kSub) - GEN("ri", CALL_C(translateSub)) - GEN("rr", CALL_C(translateSub)) - GEN("rm", CALL_C(translateSub)) - GEN("Rri", CALL_C(translateSub)) - GEN("Rrr", CALL_C(translateSub)) - GEN("Rrm", CALL_C(translateSub)) -END_RULES - -BEGIN_RULES(Instruction::kAnd) - GEN("ri", CALL_C(translateAnd)) - GEN("rr", CALL_C(translateAnd)) - GEN("rm", CALL_C(translateAnd)) - GEN("Rri", CALL_C(translateAnd)) - GEN("Rrr", CALL_C(translateAnd)) - GEN("Rrm", CALL_C(translateAnd)) -END_RULES - -BEGIN_RULES(Instruction::kOr) - GEN("ri", CALL_C(translateOr)) - GEN("rr", CALL_C(translateOr)) - GEN("rm", CALL_C(translateOr)) - GEN("Rri", CALL_C(translateOr)) - GEN("Rrr", CALL_C(translateOr)) - GEN("Rrm", CALL_C(translateOr)) -END_RULES - -BEGIN_RULES(Instruction::kXor) - GEN("ri", CALL_C(translateXor)) - GEN("rr", CALL_C(translateXor)) - GEN("rm", CALL_C(translateXor)) - GEN("Rri", CALL_C(translateXor)) - GEN("Rrr", CALL_C(translateXor)) - GEN("Rrm", CALL_C(translateXor)) -END_RULES - -BEGIN_RULES(Instruction::kMul) - GEN("ri", CALL_C(translateMul)) - GEN("rr", CALL_C(translateMul)) - GEN("rm", CALL_C(translateMul)) - GEN("Rri", CALL_C(translateMul)) - GEN("Rrr", CALL_C(translateMul)) - GEN("Rrm", CALL_C(translateMul)) -END_RULES - -BEGIN_RULES(Instruction::kDiv) - GEN("rrr", CALL_C(translateDiv)) - GEN("rrm", CALL_C(translateDiv)) - GEN("rr", CALL_C(translateDiv)) - GEN("rm", CALL_C(translateDiv)) -END_RULES - -BEGIN_RULES(Instruction::kDivUn) - GEN("rrr", CALL_C(translateDivUn)) - GEN("rrm", CALL_C(translateDivUn)) - GEN("rr", CALL_C(translateDivUn)) - GEN("rm", CALL_C(translateDivUn)) -END_RULES - -BEGIN_RULES(Instruction::kFadd) - GEN("Xxx", ASM(fadd, OP(0), OP(1), OP(2))) - GEN("xx", ASM(fadd, OP(0), OP(0), OP(1))) -END_RULES - -BEGIN_RULES(Instruction::kFsub) - GEN("Xxx", ASM(fsub, OP(0), OP(1), OP(2))) - GEN("xx", ASM(fsub, OP(0), OP(0), OP(1))) -END_RULES - -BEGIN_RULES(Instruction::kFmul) - GEN("Xxx", ASM(fmul, OP(0), OP(1), OP(2))) - GEN("xx", ASM(fmul, OP(0), OP(0), OP(1))) -END_RULES - -BEGIN_RULES(Instruction::kFdiv) - GEN("Xxx", ASM(fdiv, OP(0), OP(1), OP(2))) - GEN("xx", ASM(fdiv, OP(0), OP(0), OP(1))) -END_RULES - -BEGIN_RULES(Instruction::kPush) - GEN("r", CALL_C(translatePush)) - GEN("m", CALL_C(translatePush)) - GEN("i", CALL_C(translatePush)) -END_RULES - -BEGIN_RULES(Instruction::kPop) - GEN("R", CALL_C(translatePop)) - GEN("M", CALL_C(translatePop)) -END_RULES - -BEGIN_RULES(Instruction::kExchange) - GEN("Rr", CALL_C(translateExchange)) - GEN("Xx", CALL_C(translateExchange)) -END_RULES - -BEGIN_RULES(Instruction::kCmp) - GEN("rr", CALL_C(translateCmp)) - GEN("ri", CALL_C(translateCmp)) - GEN("xx", CALL_C(translateCmp)) -END_RULES - -BEGIN_RULES(Instruction::kTest) - GEN("rr", CALL_C(translateTst)) -END_RULES - -BEGIN_RULES(Instruction::kTest32) - GEN("rr", ASM(tst, REG_OP(0, 32), REG_OP(1, 32))) -END_RULES - -BEGIN_RULES(Instruction::kBranch) - GEN("b", ASM(b, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchZ) - GEN("b", ASM(b_eq, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchNZ) - GEN("b", ASM(b_ne, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchA) - GEN("b", ASM(b_hi, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchB) - GEN("b", ASM(b_lo, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchAE) - GEN("b", ASM(b_hs, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchBE) - GEN("b", ASM(b_ls, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchG) - GEN("b", ASM(b_gt, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchL) - GEN("b", ASM(b_lt, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchGE) - GEN("b", ASM(b_ge, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchLE) - GEN("b", ASM(b_le, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchC) - GEN("b", ASM(b_cs, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchNC) - GEN("b", ASM(b_cc, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchO) - GEN("b", ASM(b_vs, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchNO) - GEN("b", ASM(b_vc, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchS) - GEN("b", ASM(b_mi, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchNS) - GEN("b", ASM(b_pl, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchE) - GEN("b", ASM(b_eq, LBL(0))) -END_RULES - -BEGIN_RULES(Instruction::kBranchNE) - GEN("b", ASM(b_ne, LBL(0))) -END_RULES - -#define DEF_COMPARE_OP_RULES(name, fpcomp) \ -BEGIN_RULES(Instruction::name) \ - GEN("Rrr", CALL_C(TranslateCompare)) \ - GEN("Rri", CALL_C(TranslateCompare)) \ - GEN("Rrm", CALL_C(TranslateCompare)) \ - if (fpcomp) { \ - GEN("Rxx", CALL_C(TranslateCompare)) \ - } \ -END_RULES - -DEF_COMPARE_OP_RULES(kEqual, true) -DEF_COMPARE_OP_RULES(kNotEqual, true) -DEF_COMPARE_OP_RULES(kGreaterThanUnsigned, true) -DEF_COMPARE_OP_RULES(kGreaterThanEqualUnsigned, true) -DEF_COMPARE_OP_RULES(kLessThanUnsigned, true) -DEF_COMPARE_OP_RULES(kLessThanEqualUnsigned, true) -DEF_COMPARE_OP_RULES(kGreaterThanSigned, false) -DEF_COMPARE_OP_RULES(kGreaterThanEqualSigned, false) -DEF_COMPARE_OP_RULES(kLessThanSigned, false) -DEF_COMPARE_OP_RULES(kLessThanEqualSigned, false) - -#undef DEF_COMPARE_OP_RULES - -BEGIN_RULES(Instruction::kInc) - GEN("r", CALL_C(translateInc)) - GEN("m", CALL_C(translateInc)) -END_RULES - -BEGIN_RULES(Instruction::kDec) - GEN("r", CALL_C(translateDec)) - GEN("m", CALL_C(translateDec)) -END_RULES - -BEGIN_RULES(Instruction::kBitTest) - GEN("ri", CALL_C(translateBitTest)); -END_RULES - -BEGIN_RULES(Instruction::kYieldInitial) - GEN(ANY, CALL_C(translateYieldInitial)) -END_RULES - -#if PY_VERSION_HEX < 0x030C0000 -BEGIN_RULES(Instruction::kYieldFrom) - GEN(ANY, CALL_C(translateYieldFrom)) -END_RULES -#else -// In 3.12+ YieldFrom is a pseudo-op which is YieldValue plus enough -// information to know which live value contains the target iterator. See -// emitStoreGenYieldPoint() for where this is captured. The target iterator is -// used for things like the result of reading gi_yieldfrom. -BEGIN_RULES(Instruction::kYieldFrom) - GEN(ANY, CALL_C(translateYieldValue)) -END_RULES #endif -BEGIN_RULES(Instruction::kYieldFromSkipInitialSend) - GEN(ANY, CALL_C(translateYieldFrom)) -END_RULES +// Translates a single LIR instruction to machine code. +void AutoTranslator::translateInstr(Environ* env, const Instruction* instr) + const { + auto opcode = instr->opcode(); + +#if defined(CINDER_AARCH64) + // Addressing frame slots through SP is only valid while SP and FP both hold + // their frame positions, so writing either one gives up on it. Generators + // are the reason this is not just a prologue/epilogue concern: they re-point + // FP at the heap-allocated GenDataFooter partway through the function. + // Translators that re-establish a known position (kSetupFrame, kPush, kPop) + // do so after this runs. + if (instr->getNumOutputs() > 0) { + const lir::Operand* out = instr->output(); + if (out->isReg()) { + auto loc = out->getPhyRegister(); + if (loc == SP || loc == arch::reg_frame_pointer_loc) { + env->sp_to_fp_delta = arch::kSpPositionUnknown; + } + } + } +#endif + + // Every conditional branch reads its condition out of the status flags and + // jumps to the label in its first input, so they all lower the same way. + if (opcode == Opcode::kBranchCC) { + emitBranchCC( + env->as, instr->condition(), getLabel(env, instr->getInput(0))); + return; + } + + switch (opcode) { + case Opcode::kBind: + case Opcode::kCallSiteLiveValues: + return; +#if defined(CINDER_X86_64) + case Opcode::kLea: { + auto* output = instr->output(); + auto* input = instr->getInput(0); -BEGIN_RULES(Instruction::kYieldFromHandleStopAsyncIteration) - GEN(ANY, CALL_C(translateYieldFrom)) -END_RULES + if (input->isLabel()) { + translateLeaLabel(env, instr); + } else { + env->as->lea(getReg(instr, output), getMem(instr, input)); + } + return; + } + case Opcode::kMoveRelaxed: { + checkMoveRelaxedOperandShape(instr); + + auto* output = instr->output(); + auto* input = instr->getInput(0); + + if (output->isReg()) { + int access_size_in_bytes = getOperandSizeInBytes(instr, output); + if (!kCinderJitTsanEnabled || + !tryEmitTsanRelaxedAtomicRead( + *env, output, input, access_size_in_bytes)) { + env->as->mov(getReg(instr, output), getMem(instr, input)); + } + } else if (input->isReg()) { + int access_size_in_bytes = getOperandSizeInBytes(instr, output); + if (!kCinderJitTsanEnabled || + !tryEmitTsanRelaxedAtomicWrite( + *env, output, input, access_size_in_bytes)) { + env->as->mov(getMem(instr, output), getReg(instr, input)); + } + } else { + int access_size_in_bytes = getOperandSizeInBytes(instr, output); + if (!kCinderJitTsanEnabled || + !tryEmitTsanRelaxedAtomicWrite( + *env, output, input, access_size_in_bytes)) { + env->as->mov(getMem(instr, output), getImm(input)); + } + } + return; + } + case Opcode::kZext: { + auto* output = instr->output(); + auto* input = instr->getInput(0); + + // x86-64 has no `movzx r64, r/m32`; writing a 32-bit register already + // zeroes the upper half of its 64-bit counterpart, so a plain MOV between + // the 32-bit halves is the zero-extend. This stays a MOV even when the + // source and destination registers are the same, as the upper half is not + // known to be clear. + if (input->sizeInBits() == 32) { + JIT_THROW_IF( + output->sizeInBits() != 64, + "Zero-extend from 32-bits should always go to 64-bits, got '{}' " + "instead", + *instr); -BEGIN_RULES(Instruction::kYieldValue) - GEN(ANY, CALL_C(translateYieldValue)) -END_RULES + auto output_reg = asmjit::x86::gpd(output->getPhyRegister().loc); + if (input->isReg()) { + env->as->mov(output_reg, getReg(instr, input)); + } else { + env->as->mov(output_reg, getMem(instr, input)); + } + return; + } -BEGIN_RULES(Instruction::kSelect) - GEN("Rrri", CALL_C(translateSelect)) -END_RULES + if (input->isReg()) { + env->as->movzx(getReg(instr, output), getReg(instr, input)); + } else { + env->as->movzx(getReg(instr, output), getMem(instr, input)); + } + return; + } + case Opcode::kSext: { + auto* output = instr->output(); + auto* input = instr->getInput(0); + + // x86-64 spells the 32 -> 64 bit sign-extend `movsxd`; `movsx` cannot + // encode a 32-bit source. + if (input->sizeInBits() == 32) { + JIT_THROW_IF( + output->sizeInBits() != 64, + "Sign-extend from 32-bits should always go to 64-bits, got '{}' " + "instead", + *instr); -BEGIN_RULES(Instruction::kIntToBool) - GEN("Rr", CALL_C(translateIntToBool)) - GEN("Ri", CALL_C(translateIntToBool)) -END_RULES + if (input->isReg()) { + env->as->movsxd(getReg(instr, output), getReg(instr, input)); + } else { + env->as->movsxd(getReg(instr, output), getMem(instr, input)); + } + return; + } -END_RULE_TABLE -// clang-format on -#else + if (input->isReg()) { + env->as->movsx(getReg(instr, output), getReg(instr, input)); + } else { + env->as->movsx(getReg(instr, output), getMem(instr, input)); + } + return; + } + case Opcode::kUnreachable: + env->as->ud2(); + return; + case Opcode::kDiv: { + auto numInputs = instr->getNumInputs(); + auto* in0 = instr->getInput(0); + auto* in1 = instr->getInput(1); + + if (numInputs == 3) { + auto* in2 = instr->getInput(2); + + if (in2->isReg()) { + env->as->idiv( + getReg(instr, in0), getReg(instr, in1), getReg(instr, in2)); + } else { + env->as->idiv( + getReg(instr, in0), getReg(instr, in1), getMem(instr, in2)); + } + } else { + if (in1->isReg()) { + env->as->idiv(getReg(instr, in0), getReg(instr, in1)); + } else { + env->as->idiv(getReg(instr, in0), getMem(instr, in1)); + } + } + return; + } + case Opcode::kDivUn: { + auto numInputs = instr->getNumInputs(); + auto* in0 = instr->getInput(0); + auto* in1 = instr->getInput(1); + + if (numInputs == 3) { + auto* in2 = instr->getInput(2); + + if (in2->isReg()) { + env->as->div( + getReg(instr, in0), getReg(instr, in1), getReg(instr, in2)); + } else { + env->as->div( + getReg(instr, in0), getReg(instr, in1), getMem(instr, in2)); + } + } else { + if (in1->isReg()) { + env->as->div(getReg(instr, in0), getReg(instr, in1)); + } else { + env->as->div(getReg(instr, in0), getMem(instr, in1)); + } + } + return; + } + case Opcode::kPush: { + auto* input = instr->getInput(0); + + if (input->isReg()) { + env->as->push(getReg(instr, input)); + } else if (input->isImm()) { + env->as->push(getImm(input)); + } else { + env->as->push(getMem(instr, input)); + } + return; + } + case Opcode::kPop: { + auto* output = instr->output(); + + if (output->isReg()) { + env->as->pop(getReg(instr, output)); + } else { + env->as->pop(getMem(instr, output)); + } + return; + } + case Opcode::kX64Cdq: { + auto* output = instr->output(); + auto* input = instr->getInput(0); + + env->as->cdq(getReg(instr, output), getReg(instr, input)); + return; + } + case Opcode::kX64Cwd: { + auto* output = instr->output(); + auto* input = instr->getInput(0); + + env->as->cwd(getReg(instr, output), getReg(instr, input)); + return; + } + case Opcode::kX64Cqo: { + auto* output = instr->output(); + auto* input = instr->getInput(0); + + env->as->cqo(getReg(instr, output), getReg(instr, input)); + return; + } + case Opcode::kTest: { + auto* in0 = instr->getInput(0); + auto* in1 = instr->getInput(1); + + env->as->test(getReg(instr, in0), getReg(instr, in1)); + return; + } + case Opcode::kBranch: { + auto* input = instr->getInput(0); + if (input->isInd() || input->isReg()) { + translateBranchIndirect(env, instr); + } else if (input->isImm()) { + env->as->jmp(getImm(input)); + } else { + env->as->jmp(getLabel(env, input)); + } + return; + } + case Opcode::kGuard: + translateGuard(env, instr); + return; + case Opcode::kDeoptPatchpoint: + TranslateDeoptPatchpoint(env, instr); + return; + case Opcode::kLoadThreadState: + translateLoadThreadState(env, instr); + return; + case Opcode::kStoreGenYieldPoint: + translateStoreGenYieldPoint(env, instr); + return; + case Opcode::kStoreGenYieldFromPoint: + translateStoreGenYieldFromPoint(env, instr); + return; + case Opcode::kBranchToYieldExit: + JIT_ABORT("kBranchToYieldExit should have been removed by regalloc"); + case Opcode::kResumeGenYield: + translateResumeGenYield(env, instr); + return; + case Opcode::kEpilogueEnd: + translateEpilogueEnd(env, instr); + return; + case Opcode::kIntToBool: + translateIntToBool(env, instr); + return; + case Opcode::kPrologue: + translatePrologue(env, instr); + return; + case Opcode::kSetupFrame: + translateSetupFrame(env, instr); + return; + case Opcode::kInc: { + auto* input = instr->getInput(0); + + if (input->isStack()) { + env->as->inc(getMem(instr, input)); + } else { + env->as->inc(getReg(instr, input)); + } + return; + } + case Opcode::kDec: { + auto* input = instr->getInput(0); + + if (input->isStack()) { + env->as->dec(getMem(instr, input)); + } else { + env->as->dec(getReg(instr, input)); + } + return; + } + case Opcode::kBranchBitSet: + case Opcode::kBranchBitNotSet: { + auto* in0 = instr->getInput(0); + auto* in1 = instr->getInput(1); + auto label = getLabel(env, instr->getInput(2)); + + env->as->bt(getReg(instr, in0), getImm(in1)); + if (instr->isBranchBitSet()) { + env->as->jc(label); + } else { + env->as->jnc(label); + } + return; + } + case Opcode::kSelect: { + auto output = getReg(instr, instr->output()); + auto condition = getReg(instr, instr->getInput(0)); + auto false_val = instr->getInput(2); + + if (false_val->isImm()) { + env->as->mov(output, getImm(false_val)); + } else { + env->as->mov(output, getReg(instr, false_val)); + } + env->as->test(condition, condition); + env->as->cmovnz(output, getReg(instr, instr->getInput(1))); + return; + } + case Opcode::kCompare: + TranslateCompare(env, instr); + return; + case Opcode::kFadd: { + if (instr->getNumOutputs() > 0) { + env->as->movsd(getVecD(instr->output()), getVecD(instr->getInput(0))); + env->as->addsd(getVecD(instr->output()), getVecD(instr->getInput(1))); + } else { + env->as->addsd( + getVecD(instr->getInput(0)), getVecD(instr->getInput(1))); + } + return; + } + case Opcode::kFsub: { + if (instr->getNumOutputs() > 0) { + env->as->movsd(getVecD(instr->output()), getVecD(instr->getInput(0))); + env->as->subsd(getVecD(instr->output()), getVecD(instr->getInput(1))); + } else { + env->as->subsd( + getVecD(instr->getInput(0)), getVecD(instr->getInput(1))); + } + return; + } + case Opcode::kFmul: { + if (instr->getNumOutputs() > 0) { + env->as->movsd(getVecD(instr->output()), getVecD(instr->getInput(0))); + env->as->mulsd(getVecD(instr->output()), getVecD(instr->getInput(1))); + } else { + env->as->mulsd( + getVecD(instr->getInput(0)), getVecD(instr->getInput(1))); + } + return; + } + case Opcode::kFdiv: { + if (instr->getNumOutputs() > 0) { + env->as->movsd(getVecD(instr->output()), getVecD(instr->getInput(0))); + env->as->divsd(getVecD(instr->output()), getVecD(instr->getInput(1))); + } else { + env->as->divsd( + getVecD(instr->getInput(0)), getVecD(instr->getInput(1))); + } + return; + } + case Opcode::kExchange: { + auto* output = instr->output(); + auto* input = instr->getInput(0); + + if (output->isVecD()) { + auto left = getVecD(output); + auto right = getVecD(input); + + env->as->pxor(left, right); + env->as->pxor(right, left); + env->as->pxor(left, right); + } else { + env->as->xchg(getReg(instr, output), getReg(instr, input)); + } + return; + } + case Opcode::kCmp: { + auto* in0 = instr->getInput(0); + auto* in1 = instr->getInput(1); + + if (in0->isVecD()) { + env->as->comisd(getVecD(in0), getVecD(in1)); + } else if (in1->isImm()) { + env->as->cmp(getReg(instr, in0), getImm(in1)); + } else { + env->as->cmp(getReg(instr, in0), getReg(instr, in1)); + } + return; + } + case Opcode::kNegate: { + if (instr->getNumOutputs() == 0) { + env->as->neg(getReg(instr, instr->getInput(0))); + } else { + auto* output = instr->output(); + auto* input = instr->getInput(0); + + if (input->isImm()) { + env->as->mov( + getReg(instr, output), asmjit::Imm(-input->getConstant())); + } else { + if (input->isStack()) { + env->as->mov(getReg(instr, output), getMem(instr, input)); + } else { + env->as->mov(getReg(instr, output), getReg(instr, input)); + } + env->as->neg(getReg(instr, output)); + } + } + return; + } + case Opcode::kInvert: { + auto* output = instr->output(); + auto* input = instr->getInput(0); + + if (input->isImm()) { + env->as->mov(getReg(instr, output), asmjit::Imm(~input->getConstant())); + } else { + if (input->isStack()) { + env->as->mov(getReg(instr, output), getMem(instr, input)); + } else { + env->as->mov(getReg(instr, output), getReg(instr, input)); + } + env->as->not_(getReg(instr, output)); + } + return; + } + case Opcode::kAdd: + case Opcode::kSub: + case Opcode::kAnd: + case Opcode::kOr: + case Opcode::kXor: + case Opcode::kMul: { + auto emitOp = [&](const auto& dst, const auto& src) { + // NOLINTNEXTLINE(clang-diagnostic-switch-enum) + switch (opcode) { + case Opcode::kAdd: + env->as->add(dst, src); + break; + case Opcode::kSub: + env->as->sub(dst, src); + break; + case Opcode::kAnd: + env->as->and_(dst, src); + break; + case Opcode::kOr: + env->as->or_(dst, src); + break; + case Opcode::kXor: + env->as->xor_(dst, src); + break; + case Opcode::kMul: + env->as->imul(dst, src); + break; + default: + JIT_ABORT("unexpected opcode"); + } + }; + + if (instr->getNumOutputs() > 0) { + auto* output = instr->output(); + auto* in0 = instr->getInput(0); + auto* in1 = instr->getInput(1); + + env->as->mov(getReg(instr, output), getReg(instr, in0)); + if (in1->isImm()) { + emitOp(getReg(instr, output), getImm(in1)); + } else if (in1->isStack()) { + emitOp(getReg(instr, output), getMem(instr, in1)); + } else { + emitOp(getReg(instr, output), getReg(instr, in1)); + } + } else { + auto* in0 = instr->getInput(0); + auto* in1 = instr->getInput(1); + + if (in1->isImm()) { + emitOp(getReg(instr, in0), getImm(in1)); + } else if (in1->isStack()) { + emitOp(getReg(instr, in0), getMem(instr, in1)); + } else { + emitOp(getReg(instr, in0), getReg(instr, in1)); + } + } + return; + } + case Opcode::kLShift: + case Opcode::kRShift: + case Opcode::kRShiftUn: + translateShift(env, instr); + return; + case Opcode::kTest32: { + auto* in0 = instr->getInput(0); + auto* in1 = instr->getInput(1); + + env->as->test( + asmjit::x86::gpd(in0->getPhyRegister().loc), + asmjit::x86::gpd(in1->getPhyRegister().loc)); + return; + } + case Opcode::kInt64ToDouble: { + auto* input = instr->getInput(0); + + if (input->isReg()) { + env->as->cvtsi2sd(getVecD(instr->output()), getReg(instr, input)); + } else { + env->as->cvtsi2sd(getVecD(instr->output()), getMem(instr, input)); + } + return; + } + case Opcode::kCall: { + auto* input = instr->getInput(0); + + if (input->isImm()) { + env->as->call(getImm(input)); + } else if (input->isLabel()) { + env->as->call(getLabel(env, input)); + } else if (input->isStack()) { + env->as->call(getMem(instr, input)); + } else { + env->as->call(getReg(instr, input)); + } + + asmjit::Label label = env->as->newLabel(); + env->as->bind(label); + if (instr->origin()) { + env->pending_debug_locs.emplace_back(label, instr->origin()); + } + fillCallSiteLiveValueLocations(env, instr); + return; + } + case Opcode::kMove: { + auto* output = instr->output(); + auto* input = instr->getInput(0); + + if (output->isReg() && output->isVecD()) { + if (input->isReg() && input->isVecD()) { + env->as->movsd(getVecD(output), getVecD(input)); + } else if (input->isReg()) { + env->as->movq(getVecD(output), getReg(instr, input)); + } else { + if constexpr (kCinderJitTsanEnabled) { + int access_size_in_bytes = getOperandSizeInBytes(instr, output); + emitTsanRead(*env, input, access_size_in_bytes); + } + env->as->movsd(getVecD(output), getMem(instr, input)); + } + } else if (output->isReg()) { + if (input->isReg() && input->isVecD()) { + env->as->movq(getReg(instr, output), getVecD(input)); + } else if (input->isReg()) { + env->as->mov(getReg(instr, output), getReg(instr, input)); + } else if (input->isImm()) { + env->as->mov(getReg(instr, output), getImm(input)); + } else { + if constexpr (kCinderJitTsanEnabled) { + int access_size_in_bytes = getOperandSizeInBytes(instr, output); + emitTsanRead(*env, input, access_size_in_bytes); + } + env->as->mov(getReg(instr, output), getMem(instr, input)); + } + } else { + if constexpr (kCinderJitTsanEnabled) { + int access_size_in_bytes = getOperandSizeInBytes(instr, output); + emitTsanWrite(*env, output, access_size_in_bytes); + } + if (input->isReg() && input->isVecD()) { + env->as->movsd(getMem(instr, output), getVecD(input)); + } else if (input->isReg()) { + env->as->mov(getMem(instr, output), getReg(instr, input)); + } else { + env->as->mov(getMem(instr, output), getImm(input)); + } + } + return; + } + case Opcode::kLoad: { + auto* output = instr->output(); + auto* input = instr->getInput(0); + JIT_CHECK( + output->isReg(), + "Load output must be a register, got {} in {}", + output->type(), + *instr); + JIT_CHECK( + isMemoryMoveOperand(input), + "Load input must be memory (Stk/Mem/Ind), got {} in {}", + input->type(), + *instr); + if (output->isVecD()) { + if constexpr (kCinderJitTsanEnabled) { + int access_size_in_bytes = getOperandSizeInBytes(instr, output); + emitTsanRead(*env, input, access_size_in_bytes); + } + env->as->movsd(getVecD(output), getMem(instr, input)); + } else { + if (input->isReg() && input->isVecD()) { + JIT_ABORT("Load from VecD register not supported"); + } + if constexpr (kCinderJitTsanEnabled) { + int access_size_in_bytes = getOperandSizeInBytes(instr, output); + emitTsanRead(*env, input, access_size_in_bytes); + } + env->as->mov(getReg(instr, output), getMem(instr, input)); + } + return; + } + case Opcode::kStore: { + auto* output = instr->output(); + auto* input = instr->getInput(0); + JIT_CHECK( + isMemoryMoveOperand(output), + "Store output must be memory (Stk/Mem/Ind), got {}", + output->type()); + if constexpr (kCinderJitTsanEnabled) { + int access_size_in_bytes = getOperandSizeInBytes(instr, output); + emitTsanWrite(*env, output, access_size_in_bytes); + } + if (input->isReg() && input->isVecD()) { + env->as->movsd(getMem(instr, output), getVecD(input)); + } else if (input->isReg()) { + env->as->mov(getMem(instr, output), getReg(instr, input)); + } else if (input->isImm()) { + env->as->mov(getMem(instr, output), getImm(input)); + } else { + JIT_ABORT("Store input must be Reg or Imm, got {}", input->type()); + } + return; + } + case Opcode::kReserveStack: + translateReserveStack(env, instr); + return; + case Opcode::kVariadicPush: + translateVariadicPush(env, instr); + return; + case Opcode::kStorePair: + translateStorePair(env, instr); + return; + case Opcode::kLoadPair: + translateLoadPair(env, instr); + return; + case Opcode::kLeave: + translateLeave(env); + return; + case Opcode::kRet: + translateRet(env); + return; + case Opcode::kNop: + case Opcode::kVectorCallTstate: + case Opcode::kVarArgCall: + case Opcode::kMulAdd: + case Opcode::kLoadArg: + case Opcode::kLoadSecondCallResult: + case Opcode::kMovConstPool: + case Opcode::kCmpBranchZero: + case Opcode::kCmpBranchNonZero: + case Opcode::kCondBranch: + case Opcode::kPhi: + case Opcode::kReturn: + JIT_ABORT("Unexpected opcode {} in translateInstr", (int)opcode); +#elif defined(CINDER_AARCH64) + case Opcode::kLea: { + auto* input = instr->getInput(0); + + if (input->isLabel()) { + translateLeaLabel(env, instr); + } else { + translateLea(env, instr); + } + return; + } + case Opcode::kMoveRelaxed: + translateMove(env, instr); + return; + case Opcode::kZext: + translateZext(env, instr); + return; + case Opcode::kSext: + translateSext(env, instr); + return; + case Opcode::kUnreachable: + translateUnreachable(env, instr); + return; + case Opcode::kDiv: + translateDiv(env, instr); + return; + case Opcode::kDivUn: + translateDivUn(env, instr); + return; + case Opcode::kPush: + translatePush(env, instr); + return; + case Opcode::kPop: + translatePop(env, instr); + return; + case Opcode::kTest: + translateTst(env, instr); + return; + case Opcode::kBranch: { + auto* input = instr->getInput(0); + if (input->isInd() || input->isReg()) { + translateBranchIndirect(env, instr); + } else if (input->isImm()) { + env->as->b(static_cast(input->getConstant())); + } else { + env->as->b(getLabel(env, input)); + } + return; + } + case Opcode::kCmpBranchZero: + env->as->cbz( + getGpWiden(instr->getInput(0)), getLabel(env, instr->getInput(1))); + return; + case Opcode::kCmpBranchNonZero: + env->as->cbnz( + getGpWiden(instr->getInput(0)), getLabel(env, instr->getInput(1))); + return; + case Opcode::kA64GuardCC: + translateA64GuardCC(env, instr); + return; + case Opcode::kGuard: + translateGuard(env, instr); + return; + case Opcode::kDeoptPatchpoint: + TranslateDeoptPatchpoint(env, instr); + return; + case Opcode::kLoadThreadState: + translateLoadThreadState(env, instr); + return; + case Opcode::kStoreGenYieldPoint: + translateStoreGenYieldPoint(env, instr); + return; + case Opcode::kStoreGenYieldFromPoint: + translateStoreGenYieldFromPoint(env, instr); + return; + case Opcode::kBranchToYieldExit: + JIT_ABORT("kBranchToYieldExit should have been removed by regalloc"); + case Opcode::kResumeGenYield: + translateResumeGenYield(env, instr); + return; + case Opcode::kEpilogueEnd: + translateEpilogueEnd(env, instr); + return; + case Opcode::kIntToBool: + translateIntToBool(env, instr); + return; + case Opcode::kPrologue: + translatePrologue(env, instr); + return; + case Opcode::kSetupFrame: + translateSetupFrame(env, instr); + return; + case Opcode::kInc: + translateInc(env, instr); + return; + case Opcode::kDec: + translateDec(env, instr); + return; + case Opcode::kBranchBitSet: + translateBranchBit(env, instr, true); + return; + case Opcode::kBranchBitNotSet: + translateBranchBit(env, instr, false); + return; + case Opcode::kSelect: + translateSelect(env, instr); + return; + case Opcode::kCompare: + TranslateCompare(env, instr); + return; + case Opcode::kFadd: { + auto* in0 = instr->getInput(0); + auto* in1 = instr->getInput(1); + + if (instr->getNumOutputs() > 0) { + env->as->fadd(getVecD(instr->output()), getVecD(in0), getVecD(in1)); + } else { + env->as->fadd(getVecD(in0), getVecD(in0), getVecD(in1)); + } + return; + } + case Opcode::kFsub: { + auto* in0 = instr->getInput(0); + auto* in1 = instr->getInput(1); + + if (instr->getNumOutputs() > 0) { + env->as->fsub(getVecD(instr->output()), getVecD(in0), getVecD(in1)); + } else { + env->as->fsub(getVecD(in0), getVecD(in0), getVecD(in1)); + } + return; + } + case Opcode::kFmul: { + auto* in0 = instr->getInput(0); + auto* in1 = instr->getInput(1); -BEGIN_RULE_TABLE -END_RULE_TABLE + if (instr->getNumOutputs() > 0) { + env->as->fmul(getVecD(instr->output()), getVecD(in0), getVecD(in1)); + } else { + env->as->fmul(getVecD(in0), getVecD(in0), getVecD(in1)); + } + return; + } + case Opcode::kFdiv: { + auto* in0 = instr->getInput(0); + auto* in1 = instr->getInput(1); + if (instr->getNumOutputs() > 0) { + env->as->fdiv(getVecD(instr->output()), getVecD(in0), getVecD(in1)); + } else { + env->as->fdiv(getVecD(in0), getVecD(in0), getVecD(in1)); + } + return; + } + case Opcode::kInt64ToDouble: + env->as->scvtf( + getVecD(instr->output()), getReg(instr, instr->getInput(0))); + return; + case Opcode::kExchange: + translateExchange(env, instr); + return; + case Opcode::kCmp: + translateCmp(env, instr); + return; + case Opcode::kNegate: + translateNegate(env, instr); + return; + case Opcode::kInvert: + translateInvert(env, instr); + return; + case Opcode::kAdd: + translateAdd(env, instr); + return; + case Opcode::kSub: + translateSub(env, instr); + return; + case Opcode::kAnd: + translateAnd(env, instr); + return; + case Opcode::kOr: + translateOr(env, instr); + return; + case Opcode::kXor: + translateXor(env, instr); + return; + case Opcode::kMul: + translateMul(env, instr); + return; + case Opcode::kLShift: + case Opcode::kRShift: + case Opcode::kRShiftUn: + translateShift(env, instr); + return; + case Opcode::kTest32: { + auto* in0 = instr->getInput(0); + auto* in1 = instr->getInput(1); + + env->as->tst( + asmjit::a64::w(in0->getPhyRegister().loc), + asmjit::a64::w(in1->getPhyRegister().loc)); + return; + } + case Opcode::kCall: + translateCall(env, instr); + fillCallSiteLiveValueLocations(env, instr); + return; + case Opcode::kMove: + translateMove(env, instr); + return; + case Opcode::kLoad: + translateLoad(env, instr); + return; + case Opcode::kStore: + translateStore(env, instr); + return; + case Opcode::kMovConstPool: + translateMovConstPool(env, instr); + return; + case Opcode::kMulAdd: + translateMulAdd(env, instr); + return; + case Opcode::kReserveStack: + translateReserveStack(env, instr); + return; + case Opcode::kVariadicPush: + translateVariadicPush(env, instr); + return; + case Opcode::kStorePair: + translateStorePair(env, instr); + return; + case Opcode::kLoadPair: + translateLoadPair(env, instr); + return; + case Opcode::kLeave: + translateLeave(env); + return; + case Opcode::kRet: + translateRet(env); + return; + case Opcode::kNop: + case Opcode::kVectorCallTstate: + case Opcode::kVarArgCall: + case Opcode::kLoadArg: + case Opcode::kLoadSecondCallResult: + case Opcode::kCondBranch: + case Opcode::kPhi: + case Opcode::kReturn: + JIT_ABORT( + "Unexpected opcode {} ({}) in translateInstr", + opname(opcode), + static_cast(opcode)); #endif + default: + JIT_ABORT( + "No handler for opcode {} ({})", + opname(opcode), + static_cast(opcode)); + } +} -} // namespace jit::codegen::autogen +} // namespace cinderx::jit::codegen::autogen diff --git a/cinderx/Jit/codegen/autogen.h b/cinderx/Jit/codegen/autogen.h index 2fbb956bf..7e03dd5c1 100644 --- a/cinderx/Jit/codegen/autogen.h +++ b/cinderx/Jit/codegen/autogen.h @@ -6,23 +6,9 @@ #include "cinderx/Jit/codegen/arch.h" #include "cinderx/Jit/codegen/environ.h" -#include -#include - -namespace jit::codegen::autogen { - -// this struct defines a trie tree node to support instruction -// operand type matching. -struct PatternNode { - using func_t = void (*)(Environ*, const jit::lir::Instruction*); - - std::unordered_map> next; - func_t func{nullptr}; -}; +namespace cinderx::jit::codegen::autogen { // A machine code generator from LIR. -// This class generates machine code based on a set of user-defined rules. -// See autogen.cpp file for details. class AutoTranslator { public: static AutoTranslator& getInstance() { @@ -35,31 +21,33 @@ class AutoTranslator { static arch::Gp getGp(lir::DataType data_type, unsigned int reg) { #if defined(CINDER_X86_64) switch (data_type) { - case jit::lir::OperandBase::k8bit: + case jit::lir::Operand::k8bit: return asmjit::x86::gpb(reg); - case jit::lir::OperandBase::k16bit: + case jit::lir::Operand::k16bit: return asmjit::x86::gpw(reg); - case jit::lir::OperandBase::k32bit: + case jit::lir::Operand::k32bit: return asmjit::x86::gpd(reg); - case jit::lir::OperandBase::kObject: - case jit::lir::OperandBase::k64bit: + case jit::lir::Operand::kObject: + case jit::lir::Operand::kObjectUntagged: + case jit::lir::Operand::k64bit: return asmjit::x86::gpq(reg); - case jit::lir::OperandBase::kDouble: + case jit::lir::Operand::kDouble: JIT_ABORT("incorrect register type."); } #elif defined(CINDER_AARCH64) JIT_CHECK(reg != raw(RegId::SP), "SP is not a general-purpose register"); switch (data_type) { - case jit::lir::OperandBase::k8bit: - case jit::lir::OperandBase::k16bit: + case jit::lir::Operand::k8bit: + case jit::lir::Operand::k16bit: JIT_ABORT("Unsupported register size in aarch64."); - case jit::lir::OperandBase::k32bit: + case jit::lir::Operand::k32bit: return asmjit::a64::w(reg); - case jit::lir::OperandBase::kObject: - case jit::lir::OperandBase::k64bit: + case jit::lir::Operand::kObject: + case jit::lir::Operand::kObjectUntagged: + case jit::lir::Operand::k64bit: return asmjit::a64::x(reg); - case jit::lir::OperandBase::kDouble: + case jit::lir::Operand::kDouble: JIT_ABORT("incorrect register type."); } #else @@ -68,7 +56,7 @@ class AutoTranslator { Py_UNREACHABLE(); } - static arch::Gp getGp(const lir::OperandBase* op, unsigned int reg) { + static arch::Gp getGp(const lir::Operand* op, unsigned int reg) { #if defined(CINDER_X86_64) return getGp(op->dataType(), reg); #elif defined(CINDER_AARCH64) @@ -80,15 +68,15 @@ class AutoTranslator { Py_UNREACHABLE(); } - static arch::Gp getGpOutput(const lir::OperandBase* op, unsigned int reg) { + static arch::Gp getGpOutput(const lir::Operand* op, unsigned int reg) { #if defined(CINDER_X86_64) return getGp(op->dataType(), reg); #elif defined(CINDER_AARCH64) JIT_CHECK(reg != raw(RegId::SP), "SP is not a general-purpose register"); auto data_type = op->dataType(); - if (data_type == jit::lir::OperandBase::k8bit || - data_type == jit::lir::OperandBase::k16bit) { + if (data_type == jit::lir::Operand::k8bit || + data_type == jit::lir::Operand::k16bit) { return asmjit::a64::w(reg); } return getGp(op->dataType(), reg); @@ -98,11 +86,11 @@ class AutoTranslator { Py_UNREACHABLE(); } - static arch::VecD getVecD(const jit::lir::OperandBase* op) { + static arch::VecD getVecD(const jit::lir::Operand* op) { #if defined(CINDER_X86_64) auto data_type = op->dataType(); switch (data_type) { - case jit::lir::OperandBase::kDouble: + case jit::lir::Operand::kDouble: return asmjit::x86::xmm(op->getPhyRegister().loc - VECD_REG_BASE); default: JIT_ABORT("incorrect register type."); @@ -110,7 +98,7 @@ class AutoTranslator { #elif defined(CINDER_AARCH64) auto data_type = op->dataType(); switch (data_type) { - case jit::lir::OperandBase::kDouble: + case jit::lir::Operand::kDouble: return asmjit::a64::d(op->getPhyRegister().loc - VECD_REG_BASE); default: JIT_ABORT("incorrect register type."); @@ -121,26 +109,35 @@ class AutoTranslator { Py_UNREACHABLE(); } - static arch::Gp getGp(const jit::lir::OperandBase* op) { + static arch::Gp getGp(const jit::lir::Operand* op) { return getGp(op, op->getPhyRegister().loc); } - static arch::Gp getGpOutput(const jit::lir::OperandBase* op) { + static arch::Gp getGpOutput(const jit::lir::Operand* op) { return getGpOutput(op, op->getPhyRegister().loc); } - private: - AutoTranslator() { - initTable(); + static arch::Gp getGpWiden(lir::DataType data_type, unsigned int reg) { + // AArch64 has no sub-32-bit GP registers. Values in registers are + // guaranteed to be properly zero-extended by ldrb/ldrh/cset. + // For signed operations, use the postgen sign-extension pass instead. + if constexpr (kBuildArch == Arch::kAarch64) { + if (data_type == jit::lir::Operand::k8bit || + data_type == jit::lir::Operand::k16bit) { + data_type = jit::lir::Operand::k32bit; + } + } + return getGp(data_type, reg); } - std:: - unordered_map> - instr_rule_map_; + static arch::Gp getGpWiden(const lir::Operand* op) { + return getGpWiden(op->dataType(), op->getPhyRegister().loc); + } - void initTable(); + private: + AutoTranslator() = default; DISALLOW_COPY_AND_ASSIGN(AutoTranslator); }; -} // namespace jit::codegen::autogen +} // namespace cinderx::jit::codegen::autogen diff --git a/cinderx/Jit/codegen/code_section.cpp b/cinderx/Jit/codegen/code_section.cpp index 0dd7bfa84..64c0b6ccf 100644 --- a/cinderx/Jit/codegen/code_section.cpp +++ b/cinderx/Jit/codegen/code_section.cpp @@ -4,7 +4,7 @@ #include "cinderx/Common/log.h" -namespace jit::codegen { +namespace cinderx::jit::codegen { const char* codeSectionName(CodeSection section) { switch (section) { case CodeSection::kHot: @@ -16,7 +16,8 @@ const char* codeSectionName(CodeSection section) { } CodeSection codeSectionFromName(const char* name) { - if (strcmp(name, ".text") == 0 || strcmp(name, ".addrtab") == 0) { + if (strcmp(name, ".text") == 0 || strcmp(name, ".addrtab") == 0 || + strcmp(name, ".a64stubs") == 0) { return CodeSection::kHot; } if (strcmp(name, ".coldtext") == 0) { @@ -41,4 +42,4 @@ void populateCodeSections( }); } -} // namespace jit::codegen +} // namespace cinderx::jit::codegen diff --git a/cinderx/Jit/codegen/code_section.h b/cinderx/Jit/codegen/code_section.h index fd539564f..66022a5b4 100644 --- a/cinderx/Jit/codegen/code_section.h +++ b/cinderx/Jit/codegen/code_section.h @@ -10,7 +10,7 @@ #include #include -namespace jit::codegen { +namespace cinderx::jit::codegen { enum class CodeSection { kHot, @@ -91,4 +91,4 @@ void populateCodeSections( asmjit::CodeHolder& code, void* entry); -} // namespace jit::codegen +} // namespace cinderx::jit::codegen diff --git a/cinderx/Jit/codegen/copy_graph.cpp b/cinderx/Jit/codegen/copy_graph.cpp index 9fb6f3f99..abf805996 100644 --- a/cinderx/Jit/codegen/copy_graph.cpp +++ b/cinderx/Jit/codegen/copy_graph.cpp @@ -2,16 +2,7 @@ #include "cinderx/Jit/codegen/copy_graph.h" -namespace jit::codegen { - -CopyGraph::Node::~Node() { - if (child_node.isLinked()) { - child_node.Unlink(); - } - if (leaf_node.isLinked()) { - leaf_node.Unlink(); - } -} +namespace cinderx::jit::codegen { void CopyGraph::addEdge(int from, int to) { auto parent = getNode(from); @@ -34,21 +25,27 @@ CopyGraph::Node* CopyGraph::getNode(int loc) { std::forward_as_tuple(loc)); if (pair.second) { // Every node starts as a leaf. - leaf_nodes_.PushBack(pair.first->second); + leaf_nodes_.pushBack(pair.first->second); } return &pair.first->second; } void CopyGraph::setParent(Node* child, Node* parent) { JIT_DCHECK(child != parent, "Can't make node its own parent"); - if (child->child_node.isLinked()) { - child->child_node.Unlink(); + + // Remove pre-existing parent if it exists. + if (child->parent != nullptr) { + child->parent->children.remove(*child); } + + // Set the new parent. child->parent = parent; + + // If the new parent was a leaf, it no longer is one. if (parent != nullptr) { - parent->children.PushBack(*child); - if (parent->leaf_node.isLinked()) { - parent->leaf_node.Unlink(); + parent->children.pushBack(*child); + if (parent->LeafLink::isLinked()) { + leaf_nodes_.remove(*parent); } } } @@ -99,10 +96,11 @@ std::vector CopyGraph::process() { auto node = &nodes_.begin()->second; if (inRegisterCycle(node)) { - setParent(&node->children.Front(), nullptr); + setParent(&node->children.front(), nullptr); while (node->parent != nullptr) { ops.emplace_back(Op::Kind::kExchange, node->loc, node->parent->loc); auto parent = node->parent; + parent->children.remove(*node); nodes_.erase(node->loc); node = parent; } @@ -112,9 +110,9 @@ std::vector CopyGraph::process() { ops.emplace_back(Op::Kind::kCopy, node->loc, kTempLoc); auto temp_node = getNode(kTempLoc); - auto child = &node->children.Front(); + auto child = &node->children.front(); setParent(child, temp_node); - leaf_nodes_.PushBack(*node); + leaf_nodes_.pushBack(*node); processLeafNodes(ops); } @@ -122,24 +120,25 @@ std::vector CopyGraph::process() { } void CopyGraph::processLeafNodes(std::vector& ops) { - while (!leaf_nodes_.IsEmpty()) { - auto node = &leaf_nodes_.Front(); - leaf_nodes_.PopFront(); + while (!leaf_nodes_.isEmpty()) { + auto node = &leaf_nodes_.front(); + leaf_nodes_.popFront(); auto parent = node->parent; ops.emplace_back(Op::Kind::kCopy, parent->loc, node->loc); + parent->children.remove(*node); nodes_.erase(node->loc); - if (parent->children.IsEmpty()) { + if (parent->children.isEmpty()) { if (parent->parent == nullptr) { // The parent has no parent, so this was the last copy in this chain. nodes_.erase(parent->loc); } else { // Process the parent next. - leaf_nodes_.PushFront(*parent); + leaf_nodes_.pushFront(*parent); } } } } -} // namespace jit::codegen +} // namespace cinderx::jit::codegen diff --git a/cinderx/Jit/codegen/copy_graph.h b/cinderx/Jit/codegen/copy_graph.h index 83a5fbca5..0955d759f 100644 --- a/cinderx/Jit/codegen/copy_graph.h +++ b/cinderx/Jit/codegen/copy_graph.h @@ -12,7 +12,7 @@ #include #include -namespace jit::codegen { +namespace cinderx::jit::codegen { // CopyGraph is used to generate a sequence of copies and/or exchanges to // shuffle data between registers (non-negative ints) and memory locations @@ -58,9 +58,11 @@ class CopyGraph { } private: - struct Node { + struct ChildListTag {}; + struct LeafListTag {}; + struct Node : public IntrusiveListNode, + public IntrusiveListNode { explicit Node(int loc) : loc{loc} {} - ~Node(); bool operator<(const Node& other) const { return loc < other.loc; @@ -68,12 +70,13 @@ class CopyGraph { const int loc; Node* parent{nullptr}; - IntrusiveListNode child_node; - IntrusiveListNode leaf_node; - IntrusiveList children; + IntrusiveList children; DISALLOW_COPY_AND_ASSIGN(Node); + DISALLOW_MOVE_AND_ASSIGN(Node); }; + using ChildLink = IntrusiveListNode; + using LeafLink = IntrusiveListNode; // Create or look up a node for the given location. Newly-created nodes will // automatically be added to leaf_nodes_. @@ -95,7 +98,7 @@ class CopyGraph { std::map nodes_; // All nodes with no outgoing edges (children). - IntrusiveList leaf_nodes_; + IntrusiveList leaf_nodes_; }; // the same as CopyGraph, but preserves certain types of `from` nodes. @@ -139,4 +142,4 @@ class CopyGraphWithType : public CopyGraph { std::unordered_map> from_types_; }; -} // namespace jit::codegen +} // namespace cinderx::jit::codegen diff --git a/cinderx/Jit/codegen/environ.h b/cinderx/Jit/codegen/environ.h index 18f6928d1..c0a2c7511 100644 --- a/cinderx/Jit/codegen/environ.h +++ b/cinderx/Jit/codegen/environ.h @@ -2,15 +2,20 @@ #pragma once +#include "cinderx/Common/containers.h" +#include "cinderx/Common/ref.h" #include "cinderx/Jit/codegen/annotations.h" #include "cinderx/Jit/codegen/arch.h" -#include "cinderx/Jit/containers.h" #include "cinderx/Jit/context.h" #include "cinderx/Jit/debug_info.h" +#include "cinderx/Jit/inline_cache_storage.h" #include -namespace jit::codegen { +#include +#include + +namespace cinderx::jit::codegen { struct Environ { // Metadata for annotated disassembly. @@ -21,7 +26,7 @@ struct Environ { // Modified registers. Set by VariableManager and read by generatePrologue() // and generateEpilogue(). - PhyRegisterSet changed_regs{0}; + PhyRegisterSet changed_regs{}; // The size of all data stored on the C stack: shadow frames, spilled values, // saved callee-saved registers, and space for stack arguments to called @@ -35,22 +40,46 @@ struct Environ { // on the stack. int last_callee_saved_reg_off{-1}; +#if defined(CINDER_AARCH64) + // Distance from SP to FP at the point autogen has emitted code up to, or + // arch::kSpPositionUnknown outside of an established frame. Frame slots are + // named by their (negative) FP offset but are much cheaper to address + // through SP, so translators consult this to pick a base. + // + // Always kSpPositionUnknown for generators: they re-point FP at the + // heap-allocated GenDataFooter, so FP and SP no longer address the same + // memory region and no delta between them exists. See is_generator. + int32_t sp_to_fp_delta{arch::kSpPositionUnknown}; +#endif + // Various Labels that span major sections of the function. asmjit::Label static_arg_typecheck_failed_label; asmjit::Label hard_exit_label; asmjit::Label exit_label; - asmjit::Label exit_for_yield_label; asmjit::Label gen_resume_entry_label; - - // Deopt exits. One per guard. - struct DeoptExit { - DeoptExit(size_t idx, asmjit::Label lbl, const jit::lir::Instruction* ins) - : deopt_meta_index(idx), label(lbl), instr(ins) {} - size_t deopt_meta_index; - asmjit::Label label; - const jit::lir::Instruction* instr; - }; - std::vector deopt_exits; + asmjit::Label finish_frame_setup; + asmjit::Label correct_arg_count; + asmjit::Label prologue_exit; + asmjit::Label wrapper_exit; + + // Static type check jump table, resolved after code generation. + void** static_typecheck_table{nullptr}; + std::vector> + static_typecheck_jt_entries; + + // Resume label shared between StoreGenYieldPoint and ResumeGenYield. + // Created by translateStoreGenYieldPoint, bound by + // translateResumeGenYield. + asmjit::Label pending_yield_resume_label; + + // Map from deopt metadata index to the stage 1 deopt exit LIR block. + // Populated by GenerateDeoptExitBlocks (post-regalloc), used by + // TranslateGuard/TranslateDeoptPatchpoint to branch to the correct block. + UnorderedMap deopt_exit_blocks; + + // Address of the global deopt trampoline for this function. + // Set by NativeGenerator before code generation. + void* deopt_trampoline{nullptr}; struct PendingDeoptPatcher { PendingDeoptPatcher(JumpPatcher* p, asmjit::Label pp, asmjit::Label de) @@ -67,6 +96,14 @@ struct Environ { std::vector pending_debug_locs; + // Call return-address -> post-call guard deopt-exit pairings, resolved to + // addresses after code finalization. + struct CallsiteDeoptPending { + asmjit::Label return_addr_label; + asmjit::Label deopt_exit_label; + }; + std::vector callsite_deopt_pending; + // Location of incoming arguments std::vector arg_locations; @@ -74,7 +111,6 @@ struct Environ { explicit IndirectInfo(void** indirect_ptr) : indirect(indirect_ptr) {} void** indirect; - asmjit::Label trampoline{0}; }; UnorderedMap function_indirections; @@ -87,11 +123,85 @@ struct Environ { // Runtime data for this function. jit::CodeRuntime* code_rt{nullptr}; + InlineCacheStorage& inlineCacheStorage() { +#ifdef ENABLE_PREFORK_MODEL + return ctx->inlineCacheStorage(*code_rt); +#else + JIT_DCHECK( + inline_cache_storage_ != nullptr, + "inline cache storage has already been transferred"); + return *inline_cache_storage_; +#endif + } + +#ifndef ENABLE_PREFORK_MODEL + std::unique_ptr takeInlineCacheStorage() { + JIT_DCHECK( + inline_cache_storage_ != nullptr, + "inline cache storage has already been transferred"); + return std::move(inline_cache_storage_); + } +#endif + + // Pending references to be added to CodeRuntime at end of compilation. + // During threaded compilation, we record borrowed references here without + // acquiring GIL, then flush them in batch to CodeRuntime with GIL held. + // This avoids GIL contention during codegen while keeping CodeRuntime simple. + std::vector> pending_references_; + + void addReference(BorrowedRef<> obj) { + JIT_DCHECK(obj != nullptr, "no nulls allowed"); + pending_references_.push_back(obj); + } + + // Transfer the environments owned references into the CodeRuntime + void transferReferences() { + JIT_DCHECK( + ThreadedCompileContext::canAccessSharedData(), "lock should be held"); + code_rt->setReifier(Ref<>::create(reifier)); + auto pending = std::move(pending_references_); + for (BorrowedRef<> obj : pending) { + code_rt->addReference(obj); + } + } + + // Codegen-lifetime cache mapping live-value instructions to their deopt + // metadata index. Avoids duplicate CodeRuntime entries when multiple passes + // register the same instruction. + UnorderedMap deopt_instr_cache; + + template + std::size_t addDeoptMetadata(const InstrT& instr) { + auto [it, inserted] = deopt_instr_cache.emplace(&instr, 0); + if (!inserted) { + return it->second; + } + it->second = code_rt->addRawDeoptMetadata(DeoptMetadata::fromInstr(instr)); + return it->second; + } + template void addAnnotation(T&& item, asmjit::BaseNode* start_cursor) { + if (suppress_annotations) { + return; + } annotations.add(std::forward(item), as, start_cursor); } +#if defined(CINDER_AARCH64) + void adjustSp(int32_t delta) { + if (sp_to_fp_delta != arch::kSpPositionUnknown) { + sp_to_fp_delta += delta; + } + } +#endif + + // When true, addAnnotation() calls are suppressed. Set by + // generateAssemblyBody() while a text annotation is active so that + // translator-internal annotations (e.g. "Set up frame pointer") don't + // conflict with the higher-level text annotation. + bool suppress_annotations{false}; + // Map of GenYieldPoints which need their resume_target_ setting after code- // gen is complete. UnorderedMap unresolved_gen_entry_labels; @@ -109,7 +219,7 @@ struct Environ { // generation purposes. // // This is a hack. Need to do the real copy propagation after LIR cleanup is - // done. Related to jit::lir::LIRGenerator::AnalyzeCopies(). + // done. Related to jit::lir::LIRGenerator::analyzeCopies(). UnorderedMap copy_propagation_map; UnorderedMap block_label_map; @@ -117,12 +227,64 @@ struct Environ { UnorderedMap inline_frame_map; - FrameMode frame_mode; - int initial_yield_spill_size_{-1}; + struct CallSiteLiveValueMetadata { + std::size_t deopt_meta_index{}; + jit::lir::Instruction* live_values_instr{nullptr}; + }; + UnorderedMap + callsite_live_value_metadata; int max_arg_buffer_size{0}; + // Size of stack space reserved by ReserveStack instructions. This space + // is placed above the call argument buffer (at SP+max_arg_buffer_size), + // so that call args remain at SP+0 where the callee expects them per the + // ABI, and calls don't clobber the reserved data. The LIR ReserveStack + // instruction is lowered to a LEA in autogen using max_arg_buffer_size + // as the offset, and reserve_stack_size is added to the frame's arg + // buffer in computeFrameInfo. + int reserve_stack_size{0}; + bool has_inlined_functions{false}; + +#if defined(CINDER_X86_64) && defined(_WIN32) + // Offset from RBP to a 16-byte buffer used for receiving struct return + // values from C++ helper functions via the Windows x64 hidden-pointer ABI. + // Computed before LIR generation and reserved in the register allocator's + // frame so it doesn't conflict with spill slots. + int win_struct_ret_offset{0}; +#endif + + // True if the function has any DeoptBase instructions in its final HIR. + // When false, the interpreter frame can never be materialized, enabling a + // cheaper inline frame unlink at exit. + bool can_deopt{true}; + + // True if the function being compiled is a generator or coroutine. + bool is_generator{false}; + +#if defined(CINDER_AARCH64) + // Constant pool for large immediate values. translateMovConstPool populates + // these; gen_asm.cpp emits the pool data after deopt exits. + UnorderedMap const_pool_labels; +#endif + + // Frame layout computed after register allocation. Read by the kSetupFrame + // autogen translator for both the normal entry and generator resume entry. + int resume_frame_total_size{0}; + int resume_header_and_spill_size{0}; + PhyRegisterSet resume_saved_regs{}; + + // Byte offset of gi_jit_data within a generator object, computed per + // function. Read by the resume entry block builder. + Py_ssize_t gi_jit_data_offset{0}; + + BorrowedRef<> reifier; + +#ifndef ENABLE_PREFORK_MODEL + std::unique_ptr inline_cache_storage_{ + std::make_unique()}; +#endif }; -} // namespace jit::codegen +} // namespace cinderx::jit::codegen diff --git a/cinderx/Jit/codegen/frame_asm.cpp b/cinderx/Jit/codegen/frame_asm.cpp deleted file mode 100644 index 0092801d7..000000000 --- a/cinderx/Jit/codegen/frame_asm.cpp +++ /dev/null @@ -1,1157 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. - -#include "cinderx/Jit/codegen/frame_asm.h" - -#if PY_VERSION_HEX >= 0x030E0000 -#include "internal/pycore_interp_structs.h" -#endif -#include "internal/pycore_pystate.h" - -#include "cinderx/Common/py-portability.h" -#include "cinderx/Common/util.h" -#include "cinderx/Jit/codegen/arch.h" -#include "cinderx/Jit/codegen/register_preserver.h" -#include "cinderx/Jit/frame.h" -#include "cinderx/Jit/frame_header.h" -#include "cinderx/Jit/hir/type.h" -#include "cinderx/Jit/jit_rt.h" -#include "cinderx/Jit/threaded_compile.h" - -#if PY_VERSION_HEX < 0x030C0000 -#include "cinder/exports.h" -#include "internal/pycore_shadow_frame.h" -#endif - -using namespace asmjit; -using namespace jit::hir; - -namespace jit::codegen { - -#ifdef ENABLE_SHADOW_FRAMES - -namespace shadow_frame { -// Shadow stack frames appear at the beginning of native frames for jitted -// functions - -#if defined(CINDER_X86_64) -constexpr arch::Mem kFramePtr = x86::ptr(x86::rbp, -kJITShadowFrameSize); -constexpr arch::Mem kInFramePrevPtr = - x86::ptr(x86::rbp, -kJITShadowFrameSize + SHADOW_FRAME_FIELD_OFF(prev)); -constexpr arch::Mem kInFrameDataPtr = - x86::ptr(x86::rbp, -kJITShadowFrameSize + SHADOW_FRAME_FIELD_OFF(data)); -constexpr arch::Mem kInFrameOrigDataPtr = x86::ptr( - x86::rbp, - -kJITShadowFrameSize + JIT_SHADOW_FRAME_FIELD_OFF(orig_data)); - -constexpr arch::Mem getStackTopPtr(arch::Gp tstate_reg) { - return x86::ptr(tstate_reg, offsetof(PyThreadState, shadow_frame)); -} -#else -CINDER_UNSUPPORTED -#endif - -} // namespace shadow_frame - -#endif // ENABLE_SHADOW_FRAMES - -#if PY_VERSION_HEX >= 0x030C0000 - -bool tstate_offset_inited; -int32_t tstate_offset = -1; - -void initThreadStateOffset() { - if (tstate_offset_inited) { - return; - } - -#if defined(CINDER_X86_64) - // PyThreadState_GetCurrent just accesses the thread local value, and - // we want to figure out what the offset from the fs register it's - // stored at. So verify that we recognize what it's doing and pull - // out that offset. - uint8_t* ts_func = reinterpret_cast(&_PyThreadState_GetCurrent); - - // 0x4 8b 48 64 e5 89 48 55 - if (ts_func[0] == 0x55 && // push rbp - ts_func[1] == 0x48 && ts_func[2] == 0x89 && - ts_func[3] == 0xe5 && // mov rsp, rbp - ts_func[4] == 0x64 && ts_func[5] == 0x48 && ts_func[6] == 0x8b && - ts_func[7] == 0x04 && ts_func[8] == 0x25) { // movq %fs:OFFSET, %rax - // movq %fs:-0x18, %rax - tstate_offset = *reinterpret_cast(ts_func + 9); - } else { -#ifndef Py_DEBUG - assert(false); -#endif - } -#elif defined(CINDER_AARCH64) - // PyThreadState_GetCurrent just accesses the thread local value, and - // we want to figure out what the offset from the thread-local storage it's - // stored at. So verify that we recognize what it's doing and pull - // out that offset. - uint32_t* ts_func = reinterpret_cast(&_PyThreadState_GetCurrent); - - if (ts_func[0] == 0xa9bf7bfd && // stp x29, x30, [sp, #-16]! - ts_func[1] == 0x910003fd && // mov x29, sp - ((ts_func[2] & ~0x1f) == 0xd53bd048) // mrs x?, tpidr_el0 - ) { - // Here we know we are loading the thread local base offset into some - // register, based on the mrs instruction. - uint32_t reg = ts_func[2] & 0x1f; - int32_t current_offset = 0; - - // Now, we will interpret any subsequent add instructions in order to - // determine the offset. We will know we are done when we hit an ldr x0, or - // we hit something unknown and need to break. - for (size_t index = 3;; index++) { - if (ts_func[index] == (0xf9400000 | (reg << 5))) { - // ldr x0, [x?] - // - // Here we are loading the temporarily calculated offset into x0, which - // is the return register. At this point we are done. - break; - } else if ( - (ts_func[index] & ~0x7ffc00) == (0x91000000 | (reg << 5) | reg)) { - // add x?, x?, #{, } - // - // Here we are adding to the temporary offset register. It is encoded - // as: 100100010, where shift is 1 bit, imm is 12 - // bits, rn and rd are both 5 bits, which should be equivalent to reg. - uint32_t imm = (ts_func[index] >> 10) & 0xfff; - if (ts_func[index] & (1 << 22)) { - imm <<= 12; - } - - current_offset += imm; - } else { - // Otherwise, we found something we did not anticipate, so we need to - // bail out. - current_offset = -1; - break; - } - } - - tstate_offset = current_offset; - } - -#ifndef Py_DEBUG - if (tstate_offset == -1) { - assert(false); - } -#endif -#else - CINDER_UNSUPPORTED -#endif - - tstate_offset_inited = true; -} - -void FrameAsm::loadTState(const arch::Gp& dst_reg) { -#if defined(CINDER_X86_64) - if (tstate_offset != -1) { - asmjit::x86::Mem tls(tstate_offset); - tls.setSegment(x86::fs); - as_->mov(dst_reg, tls); - } else { - as_->call(_PyThreadState_GetCurrent); - as_->mov(dst_reg, x86::rax); - } -#elif defined(CINDER_AARCH64) - if (tstate_offset != -1) { - as_->mrs(dst_reg, a64::Predicate::SysReg::kTPIDR_EL0); - as_->ldr( - dst_reg, - arch::ptr_resolve(as_, dst_reg, tstate_offset, arch::reg_scratch_0)); - } else { - as_->mov(arch::reg_scratch_br, _PyThreadState_GetCurrent); - as_->blr(arch::reg_scratch_br); - as_->mov(dst_reg, a64::x0); - } -#else - CINDER_UNSUPPORTED -#endif -} - -void FrameAsm::linkNormalGeneratorFrame( - RegisterPreserver& preserver, - const arch::Gp&, - const arch::Gp& tstate_reg) { - preserver.preserve(); - -#if defined(CINDER_X86_64) - uint64_t full_words = env_.shadow_frames_and_spill_size / kPointerSize; - - as_->mov(x86::rsi, full_words); - as_->mov(x86::rdx, reinterpret_cast(codeRuntime())); - as_->lea(x86::rcx, x86::ptr(env_.gen_resume_entry_label)); - as_->mov(x86::r8, x86::rbp); - as_->call( - reinterpret_cast(JITRT_AllocateAndLinkGenAndInterpreterFrame)); - as_->mov(tstate_reg, x86::rax); - // tstate is now in RAX and GenDataFooter* in RDX. Swap RBP over to the - // generator data so spilled data starts getting stored there. There - // shouldn't have been any other data stored in the spilled area so far - // so no need to copy things over. - as_->mov(x86::rbp, x86::rdx); -#elif defined(CINDER_AARCH64) - uint64_t full_words = env_.shadow_frames_and_spill_size / kPointerSize; - - as_->mov(a64::x1, full_words); - as_->mov(a64::x2, reinterpret_cast(codeRuntime())); - as_->adr(a64::x3, env_.gen_resume_entry_label); - as_->mov(a64::x4, arch::fp); - as_->mov(arch::reg_scratch_br, JITRT_AllocateAndLinkGenAndInterpreterFrame); - as_->blr(arch::reg_scratch_br); - as_->mov(tstate_reg, a64::x0); - // tstate is now in x0 and GenDataFooter* in x1. Swap fp over to the - // generator data so spilled data starts getting stored there. There - // shouldn't have been any other data stored in the spilled area so far - // so no need to copy things over. - as_->mov(arch::fp, a64::x1); -#else - CINDER_UNSUPPORTED -#endif - - preserver.restore(); -} - -#ifdef Py_REF_DEBUG -PyInterpreterState* getPyInterpreterState() { - PyInterpreterState* interp; - if (jit::getThreadedCompileContext().compileRunning()) { - interp = jit::getThreadedCompileContext().interpreter(); - } else { - interp = PyInterpreterState_Get(); - } - return interp; -} -#endif - -#if defined(CINDER_X86_64) -#ifdef Py_GIL_DISABLED -void inc_ref_nogil( - arch::Builder* as, - const arch::Gp& reg, - const arch::Gp& scratch_reg, - const arch::Gp& tstate_reg) { - // For free-threaded Python, check immortality via ob_ref_local. - // Load ob_ref_local (32-bit). Note this load should be atomic with relaxed - // memory semantics, which is default on x86. - as->mov( - scratch_reg.r32(), x86::dword_ptr(reg, offsetof(PyObject, ob_ref_local))); - // Add 1 - if result is zero, object was immortal (UINT32_MAX + 1 overflows - // to 0) - as->inc(scratch_reg.r32()); - Label immortal = as->newLabel(); - as->jz(immortal); - - // Check if object is owned by current thread by comparing ob_tid with - // the current thread ID. This is equivalent to _Py_IsOwnedByCurrentThread. - // TODO: I don't have the stomach to find another scratch register for the - // ob_tid to current TID comparison, so I'm just reusing the one scratch - // register we have for now plus the stack. Should still be pretty fast on - // x86. - Label not_owned = as->newLabel(); - as->push(scratch_reg); - as->mov(scratch_reg, x86::ptr(reg, offsetof(PyObject, ob_tid))); - x86::Mem tid_mem; - tid_mem.setOffset(0); - tid_mem.setSize(sizeof(uintptr_t)); - tid_mem.setSegment(x86::fs); - as->cmp(scratch_reg, tid_mem); - as->pop(scratch_reg); - as->jne(not_owned); - - // Owned by current thread - store directly to ob_ref_local (fast path). - // Note this store should be atomic with relaxed memory semantics, which is - // default on x86. - as->mov( - x86::dword_ptr(reg, offsetof(PyObject, ob_ref_local)), scratch_reg.r32()); - Label done_incref = as->newLabel(); - as->jmp(done_incref); - - // Not owned - use atomic add to ob_ref_shared (slow path) - as->bind(not_owned); - as->lock().add( - x86::qword_ptr(reg, offsetof(PyObject, ob_ref_shared)), - 1 << _Py_REF_SHARED_SHIFT); - as->bind(done_incref); - -#ifdef Py_REF_DEBUG - as->inc( - x86::ptr( - tstate_reg, - offsetof(_PyThreadStateImpl, reftotal), - sizeof(Py_ssize_t))); -#endif - - as->bind(immortal); -} - -#else -// GILful inc-ref implementation -void inc_ref_gil( - arch::Builder* as, - const arch::Gp& reg, - const arch::Gp& scratch_reg) { - Label immortal = as->newLabel(); - as->mov(scratch_reg.r32(), x86::ptr(reg, offsetof(PyObject, ob_refcnt))); - as->inc(scratch_reg.r32()); -#if PY_VERSION_HEX >= 0x030E0000 - as->js(immortal); -#else - as->je(immortal); -#endif - // mortal - as->mov(x86::ptr(reg, offsetof(PyObject, ob_refcnt)), scratch_reg.r32()); - -#ifdef Py_REF_DEBUG - Py_ssize_t* ref_total = &getPyInterpreterState()->object_state.reftotal; - as->mov(scratch_reg, ref_total); - as->inc(x86::ptr(scratch_reg, 0, sizeof(void*))); -#endif - - as->bind(immortal); -} -#endif // Py_GIL_DISABLED - -void FrameAsm::incRef( - const arch::Gp& reg, - const arch::Gp& scratch_reg, - [[maybe_unused]] const arch::Gp& tstate_reg) { -#if defined(Py_GIL_DISABLED) - inc_ref_nogil(as_, reg, scratch_reg, tstate_reg); -#else - inc_ref_gil(as_, reg, scratch_reg); -#endif -} -#elif defined(CINDER_AARCH64) -void FrameAsm::incRef( - const arch::Gp& reg, - const arch::Gp& scratch_reg0, - const arch::Gp& scratch_reg1, - [[maybe_unused]] const arch::Gp& tstate_reg) { - Label immortal = as_->newLabel(); - -#if defined(Py_GIL_DISABLED) - // For free-threaded Python, check immortality via ob_ref_local. - // Load ob_ref_local (32-bit). Note this load should be atomic with relaxed - // memory semantics, which is default on aarch64 for regular loads. - as_->ldr( - scratch_reg0, - arch::ptr_offset( - reg, offsetof(PyObject, ob_ref_local), arch::AccessSize::k32)); - // Add 1 - if result is zero, object was immortal (UINT32_MAX + 1 overflows - // to 0) - as_->adds(scratch_reg0, scratch_reg0, 1); - as_->b_eq(immortal); - - // Check if object is owned by current thread by comparing ob_tid with - // the current thread ID. This is equivalent to _Py_IsOwnedByCurrentThread. - // On aarch64, the thread ID is stored at offset 0 from TPIDR_EL0. - Label not_owned = as_->newLabel(); - as_->ldr( - scratch_reg1.x(), - arch::ptr_offset(reg, offsetof(PyObject, ob_tid), arch::AccessSize::k64)); - as_->mrs(arch::reg_scratch_0, a64::Predicate::SysReg::kTPIDR_EL0); - as_->cmp(scratch_reg1.x(), arch::reg_scratch_0); - as_->b_ne(not_owned); - - // Owned by current thread - store directly to ob_ref_local (fast path). - // Note this store should be atomic with relaxed memory semantics, which is - // default on aarch64 for regular stores. - as_->str( - scratch_reg0, - arch::ptr_offset( - reg, offsetof(PyObject, ob_ref_local), arch::AccessSize::k32)); - Label done_incref = as_->newLabel(); - as_->b(done_incref); - - // Not owned - use atomic add to ob_ref_shared (slow path) - // On aarch64, we use ldxr/stxr loop for atomic operations - as_->bind(not_owned); - as_->add( - scratch_reg1.x(), - reg, - static_cast(offsetof(PyObject, ob_ref_shared))); - Label retry = as_->newLabel(); - as_->bind(retry); - as_->ldxr(scratch_reg0, a64::ptr(scratch_reg1.x())); - as_->add(scratch_reg0, scratch_reg0, 1 << _Py_REF_SHARED_SHIFT); - as_->stxr(arch::reg_scratch_0.w(), scratch_reg0, a64::ptr(scratch_reg1)); - as_->cbnz(arch::reg_scratch_0.w(), retry); - as_->bind(done_incref); - -#ifdef Py_REF_DEBUG - as_->ldr( - scratch_reg0, - arch::ptr_offset( - tstate_reg, - offsetof(_PyThreadStateImpl, reftotal), - arch::AccessSize::k64)); - as_->add(scratch_reg0, scratch_reg0, 1); - as_->str( - scratch_reg0, - arch::ptr_offset( - tstate_reg, - offsetof(_PyThreadStateImpl, reftotal), - arch::AccessSize::k64)); -#endif -#else - as_->ldr( - scratch_reg0, - arch::ptr_offset( - reg, offsetof(PyObject, ob_refcnt), arch::AccessSize::k32)); - as_->adds(scratch_reg0, scratch_reg0, 1); -#if PY_VERSION_HEX >= 0x030E0000 - as_->b_mi(immortal); -#else - as_->b_eq(immortal); -#endif - // mortal - as_->str( - scratch_reg0, - arch::ptr_offset( - reg, offsetof(PyObject, ob_refcnt), arch::AccessSize::k32)); - -#ifdef Py_REF_DEBUG - Py_ssize_t* ref_total = &getPyInterpreterState()->object_state.reftotal; - as_->mov(scratch_reg0.x(), reinterpret_cast(ref_total)); - as_->ldr(scratch_reg1.x(), a64::ptr(scratch_reg0.x())); - as_->add(scratch_reg1.x(), scratch_reg1.x(), 1); - as_->str(scratch_reg1.x(), a64::ptr(scratch_reg0.x())); -#endif -#endif - as_->bind(immortal); -} -#else -CINDER_UNSUPPORTED -#endif - -#if defined(CINDER_X86_64) -bool FrameAsm::storeConst( - const arch::Gp& reg, - int32_t offset, - void* val, - const arch::Gp& scratch) { - auto dest = x86::ptr(reg, offset, sizeof(void*)); - int64_t value = reinterpret_cast(val); - if (fitsSignedInt<32>(value)) { - // the value fits in the register, let the caller know we didn't - // populate scratch. - as_->mov(dest, static_cast(value)); - return true; - } - as_->mov(scratch, value); - as_->mov(dest, scratch); - return false; -} -#elif defined(CINDER_AARCH64) -bool FrameAsm::storeConst( - arch::Builder* as, - const arch::Gp& reg, - int32_t offset, - void* val, - const arch::Gp& scratch0, - const arch::Gp& scratch1) { - int64_t value = reinterpret_cast(val); - as_->mov(scratch0, value); - as_->str(scratch0, arch::ptr_resolve(as, reg, offset, scratch1)); - return false; -} -#else -CINDER_UNSUPPORTED -#endif - -void FrameAsm::linkLightWeightFunctionFrame( - RegisterPreserver& preserver, - const arch::Gp& func_reg, - const arch::Gp& tstate_reg) { -#if defined(ENABLE_LIGHTWEIGHT_FRAMES) -#if defined(CINDER_X86_64) - // Light weight function headers are allocated on the stack as: - // PyFunctionObject* func_obj - // _PyInterpreterFrame - // - // We need to initialize the f_code, f_funcobj fields of - // the frame along w/ the previous pointer. - asmjit::BaseNode* init_tstate_off_cursor = as_->cursor(); - initThreadStateOffset(); - env_.addAnnotation("Init tstate offset", init_tstate_off_cursor); - - // We have precious caller saved registers that we can trash - rax - // and r10 are the only non-argument registers, and our arguments - // are still in their initial registers. r10 we use for the extra - // args, and if we aren't preserving the stack it's not initialized - // yet, so we can use it. If we are preserving the stack (typically - // only in ASAN builds) then we'll need to preserve that as well - // after spilling and restoring the arguments around the call to - // get the thread state. - asmjit::BaseNode* load_tstate_cursor = as_->cursor(); - auto scratch = x86::gpq(INITIAL_EXTRA_ARGS_REG.loc); - if (tstate_offset == -1) { - preserver.preserve(); - } - loadTState(tstate_reg); - - if (tstate_offset == -1) { - preserver.restore(); - // and here's where we need to preserve the initial extra args reg - // too. - as_->push(scratch); - } - env_.addAnnotation("Load tstate", load_tstate_cursor); - - int frame_header_size = frameHeaderSizeExcludingSpillSpace(); -#if PY_VERSION_HEX < 0x030E0000 - PyObject* frame_reifier = cinderx::getModuleState()->frameReifier(); -#else - PyObject* frame_reifier = env_.code_rt->reifier(); -#endif - const auto ref_cnt = x86::rax; - -#define FRAME_OFFSET(NAME) \ - -frame_header_size + offsetof(_PyInterpreterFrame, NAME) + sizeof(FrameHeader) - - asmjit::BaseNode* store_func_cursor = as_->cursor(); -#if PY_VERSION_HEX >= 0x030E0000 - as_->mov(x86::ptr(x86::rbp, -frame_header_size, sizeof(void*)), 0); - env_.addAnnotation("Store rtfs state to 0", store_func_cursor); -#else - // Initialize the fields minus previous. - // Store func before the header - as_->mov(x86::ptr(x86::rbp, -frame_header_size), func_reg); - incRef(func_reg, ref_cnt, tstate_reg); - env_.addAnnotation("Store func before frame header", store_func_cursor); -#endif - - asmjit::BaseNode* store_f_code_cursor = as_->cursor(); -#if PY_VERSION_HEX >= 0x030E0000 - PyObject* executable = frame_reifier; -#else - PyObject* executable = (PyObject*)func_->code.get(); -#endif - bool needs_load = - storeConst(x86::rbp, FRAME_OFFSET(FRAME_EXECUTABLE), executable, scratch); - if (!_Py_IsImmortal(executable)) { - if (needs_load) { - // if this fit into a 32-bit value we didn't spill it into scratch - as_->mov(scratch, reinterpret_cast(executable)); - } - incRef(scratch, ref_cnt, tstate_reg); - } - env_.addAnnotation( - "Set _PyInterpreterFrame::f_executable/f_code", store_f_code_cursor); - - // Store f_funcobj as our helper frame reifier object - asmjit::BaseNode* store_f_funcobj_cursor = as_->cursor(); -#if PY_VERSION_HEX >= 0x030E0000 - as_->mov(x86::ptr(x86::rbp, FRAME_OFFSET(f_funcobj)), func_reg); - incRef(func_reg, ref_cnt, tstate_reg); -#else - storeConst(x86::rbp, FRAME_OFFSET(f_funcobj), frame_reifier, scratch); - JIT_DCHECK(_Py_IsImmortal(frame_reifier), "frame helper must be immortal"); -#endif - env_.addAnnotation( - "Set _PyInterpreterFrame::f_funcobj", store_f_funcobj_cursor); - - // Store prev_instr + tlbc_index - asmjit::BaseNode* store_prev_instr_cursor = as_->cursor(); -#if PY_VERSION_HEX >= 0x030E0000 - _Py_CODEUNIT* code = _PyCode_CODE(GetFunction()->code.get()); -#else - _Py_CODEUNIT* code = _PyCode_CODE(GetFunction()->code.get()) - 1; -#endif - storeConst(x86::rbp, FRAME_OFFSET(FRAME_INSTR), code, scratch); - env_.addAnnotation( - "Set _PyInterpreterFrame::prev_instr", store_prev_instr_cursor); -#ifdef Py_GIL_DISABLED - asmjit::BaseNode* tlbc_index_cursor = as_->cursor(); - as_->mov(x86::dword_ptr(x86::rbp, FRAME_OFFSET(tlbc_index)), 0); - env_.addAnnotation("Set TLBC index to 0", tlbc_index_cursor); -#endif - - // Store owner - asmjit::BaseNode* store_owner_cursor = as_->cursor(); - as_->mov( - x86::ptr(x86::rbp, FRAME_OFFSET(owner), sizeof(char)), - FRAME_OWNED_BY_THREAD); - env_.addAnnotation("Set _PyInterpreterFrame::owner", store_owner_cursor); - - // Get the frame that is currently linked into thread state and update - // our frames pointer back to it. - asmjit::BaseNode* get_tos_cursor = as_->cursor(); -#if PY_VERSION_HEX >= 0x030D0000 - // 3.14+ - current_frame is stored in PyThreadState.current_frame - const arch::Gp& frame_holder = tstate_reg; - // cur_frame->previous = PyThreadState.current_frame - as_->mov( - scratch, x86::ptr(tstate_reg, offsetof(PyThreadState, current_frame))); -#else - // 3.12 - current_frame is stored in PyThreadState.cframe - const arch::Gp& frame_holder = - x86::rax; // return value, we can freely use this as scratch - as_->mov(frame_holder, x86::ptr(tstate_reg, offsetof(PyThreadState, cframe))); - as_->mov(scratch, x86::ptr(frame_holder, offsetof(_PyCFrame, current_frame))); -#endif - env_.addAnnotation("Get topmost frame", get_tos_cursor); - - asmjit::BaseNode* store_prev_cursor = as_->cursor(); - // cur_frame->previous = PyThreadState.cframe.current_frame - as_->mov(x86::ptr(x86::rbp, FRAME_OFFSET(previous)), scratch); - env_.addAnnotation("Set _PyInterpreterFrame::previous", store_prev_cursor); - -#if PY_VERSION_HEX >= 0x030E0000 - asmjit::BaseNode* stack_pointer_cursor = as_->cursor(); - as_->lea(scratch, x86::ptr(x86::rbp, FRAME_OFFSET(localsplus))); - as_->mov(x86::ptr(x86::rbp, FRAME_OFFSET(stackpointer)), scratch); - env_.addAnnotation( - "Set _PyInterpreterFrame::stackpointer", stack_pointer_cursor); - - asmjit::BaseNode* locals_cursor = as_->cursor(); - as_->mov(x86::qword_ptr(x86::rbp, FRAME_OFFSET(f_locals)), 0); - env_.addAnnotation("Set _PyInterpreterFrame::f_locals", locals_cursor); -#endif - - // Then finally link in our frame to thread state - asmjit::BaseNode* update_linkage_cursor = as_->cursor(); - as_->lea(scratch, x86::ptr(x86::rbp, -frame_header_size + sizeof(PyObject*))); -#if PY_VERSION_HEX >= 0x030D0000 - // (PyThreadState.cframe|PyThreadState).current_frame = &cur_frame - as_->mov( - x86::ptr(frame_holder, offsetof(PyThreadState, current_frame)), scratch); -#else - // (PyThreadState.cframe|PyThreadState).current_frame = &cur_frame - as_->mov(x86::ptr(frame_holder, offsetof(_PyCFrame, current_frame)), scratch); -#endif - env_.addAnnotation( - "Set _PyInterpreterFrame as topmost frame", update_linkage_cursor); - - if (tstate_offset == -1) { - as_->pop(scratch); - } else { - preserver.remap(); - } -#elif defined(CINDER_AARCH64) - // Light weight function headers are allocated on the stack as: - // PyFunctionObject* func_obj - // _PyInterpererFrame - // - // We need to initialize the f_code, f_funcobj fields of - // the frame along w/ the previous pointer. - asmjit::BaseNode* init_tstate_off_cursor = as_->cursor(); - initThreadStateOffset(); - env_.addAnnotation("Init tstate offset", init_tstate_off_cursor); - - // We have some caller-saved registers that we can trash that are not also - // argument registers (X8-X18). X10 we use for the extra args, and if we - // aren't preserving the stack it's not initialized yet, so we can use it. If - // we are preserving the stack (typically only in ASAN builds) then we'll need - // to preserve that as well after spilling and restoring the arguments around - // the call to get the thread state. - asmjit::BaseNode* load_tstate_cursor = as_->cursor(); - auto scratch = a64::x(INITIAL_EXTRA_ARGS_REG.loc); - if (tstate_offset == -1) { - preserver.preserve(); - } - loadTState(tstate_reg); - - if (tstate_offset == -1) { - preserver.restore(); - // and here's where we need to preserve the initial extra args reg - // too. - as_->str(scratch, a64::ptr_pre(a64::sp, -16)); - } - env_.addAnnotation("Load tstate", load_tstate_cursor); - - int frame_header_size = frameHeaderSizeExcludingSpillSpace(); -#if PY_VERSION_HEX < 0x030E0000 - PyObject* frame_reifier = cinderx::getModuleState()->frameReifier(); -#else - PyObject* frame_reifier = env_.code_rt->reifier(); -#endif - const auto ref_cnt = a64::w9; - const auto ref_cnt_scratch = a64::w12; - -#define FRAME_OFFSET(NAME) \ - -frame_header_size + offsetof(_PyInterpreterFrame, NAME) + sizeof(FrameHeader) - - asmjit::BaseNode* store_func_cursor = as_->cursor(); -#if PY_VERSION_HEX >= 0x030E0000 - as_->sub(arch::reg_scratch_0, arch::fp, frame_header_size); - as_->str(a64::xzr, a64::ptr(arch::reg_scratch_0)); - env_.addAnnotation("Store rtfs state to 0", store_func_cursor); -#else - // Initialize the fields minus previous. - // Store func before the header - as_->sub(arch::reg_scratch_0, arch::fp, frame_header_size); - as_->str(func_reg, a64::ptr(arch::reg_scratch_0)); - incRef(func_reg, ref_cnt, ref_cnt_scratch, tstate_reg); - env_.addAnnotation("Store func before frame header", store_func_cursor); -#endif - - asmjit::BaseNode* store_f_code_cursor = as_->cursor(); -#if PY_VERSION_HEX >= 0x030E0000 - PyObject* executable = frame_reifier; -#else - PyObject* executable = (PyObject*)func_->code.get(); -#endif - storeConst( - as_, - arch::fp, - FRAME_OFFSET(FRAME_EXECUTABLE), - executable, - scratch, - arch::reg_scratch_1); - if (!_Py_IsImmortal(executable)) { - incRef(scratch, ref_cnt, ref_cnt_scratch, tstate_reg); - } - env_.addAnnotation("Set _PyInterpreterFrame::f_code", store_f_code_cursor); - - // Store f_funcobj as our helper frame reifier object - asmjit::BaseNode* store_f_funcobj_cursor = as_->cursor(); -#if PY_VERSION_HEX >= 0x030E0000 - as_->str( - func_reg, - arch::ptr_resolve( - as_, arch::fp, FRAME_OFFSET(f_funcobj), arch::reg_scratch_0)); - incRef(func_reg, ref_cnt, ref_cnt_scratch, tstate_reg); -#else - storeConst( - as_, - arch::fp, - FRAME_OFFSET(f_funcobj), - frame_reifier, - scratch, - arch::reg_scratch_1); - JIT_DCHECK(_Py_IsImmortal(frame_reifier), "frame helper must be immortal"); -#endif - env_.addAnnotation( - "Set _PyInterpreterFrame::f_funcobj", store_f_funcobj_cursor); - - // Store prev_instr + tlbc_index - asmjit::BaseNode* store_prev_instr_cursor = as_->cursor(); -#if PY_VERSION_HEX >= 0x030E0000 - _Py_CODEUNIT* code = _PyCode_CODE(GetFunction()->code.get()); -#else - _Py_CODEUNIT* code = _PyCode_CODE(GetFunction()->code.get()) - 1; -#endif - storeConst( - as_, - arch::fp, - FRAME_OFFSET(FRAME_INSTR), - code, - scratch, - arch::reg_scratch_1); - env_.addAnnotation( - "Set _PyInterpreterFrame::prev_instr", store_prev_instr_cursor); -#ifdef Py_GIL_DISABLED - asmjit::BaseNode* tlbc_index_cursor = as_->cursor(); - as_->str(a64::xzr, arch::ptr_offset(arch::fp, FRAME_OFFSET(tlbc_index))); - env_.addAnnotation("Set TLBC index to 0", tlbc_index_cursor); -#endif - - // Store owner - asmjit::BaseNode* store_owner_cursor = as_->cursor(); - as_->mov(a64::w1, FRAME_OWNED_BY_THREAD); - as_->strb( - a64::w1, - arch::ptr_resolve( - as_, - arch::fp, - FRAME_OFFSET(owner), - arch::reg_scratch_1, - arch::AccessSize::k32)); - env_.addAnnotation("Set _PyInterpreterFrame::owner", store_owner_cursor); - - // Get the frame that is currently linked into thread state and update - // our frames pointer back to it. - asmjit::BaseNode* get_tos_cursor = as_->cursor(); -#if PY_VERSION_HEX >= 0x030D0000 - // 3.14+ - current_frame is stored in PyThreadState.current_frame - const arch::Gp& frame_holder = tstate_reg; - // cur_frame->previous = PyThreadState.current_frame - as_->ldr( - scratch, - arch::ptr_offset(tstate_reg, offsetof(PyThreadState, current_frame))); -#else - // 3.12 - current_frame is stored in PyThreadState.cframe - const arch::Gp& frame_holder = arch::reg_scratch_0; - as_->ldr( - frame_holder, - arch::ptr_offset(tstate_reg, offsetof(PyThreadState, cframe))); - as_->ldr( - scratch, - arch::ptr_offset(frame_holder, offsetof(_PyCFrame, current_frame))); -#endif - env_.addAnnotation("Get topmost frame", get_tos_cursor); - - asmjit::BaseNode* store_prev_cursor = as_->cursor(); - // cur_frame->previous = PyThreadState.cframe.current_frame - as_->str( - scratch, - arch::ptr_resolve( - as_, arch::fp, FRAME_OFFSET(previous), arch::reg_scratch_1)); - env_.addAnnotation("Set _PyInterpreterFrame::previous", store_prev_cursor); - -#if PY_VERSION_HEX >= 0x030E0000 - asmjit::BaseNode* stack_pointer_cursor = as_->cursor(); - if (arm::Utils::isAddSubImm(-FRAME_OFFSET(localsplus))) { - as_->sub(scratch, arch::fp, -FRAME_OFFSET(localsplus)); - } else { - as_->mov(scratch, -FRAME_OFFSET(localsplus)); - as_->sub(scratch, arch::fp, scratch); - } - as_->str( - scratch, - arch::ptr_resolve( - as_, arch::fp, FRAME_OFFSET(stackpointer), arch::reg_scratch_1)); - env_.addAnnotation( - "Set _PyInterpreterFrame::stackpointer", stack_pointer_cursor); - - asmjit::BaseNode* locals_cursor = as_->cursor(); - as_->str( - a64::xzr, - arch::ptr_resolve( - as_, arch::fp, FRAME_OFFSET(f_locals), arch::reg_scratch_1)); - env_.addAnnotation("Set _PyInterpreterFrame::f_locals", locals_cursor); -#endif - - // Then finally link in our frame to thread state - asmjit::BaseNode* update_linkage_cursor = as_->cursor(); - int size = -frame_header_size + sizeof(PyObject*); - if (size > 0) { - as_->add(scratch, arch::fp, size); - } else { - as_->sub(scratch, arch::fp, -size); - } - -#if PY_VERSION_HEX >= 0x030D0000 - // (PyThreadState.cframe|PyThreadState).current_frame = &cur_frame - as_->str( - scratch, - arch::ptr_offset(frame_holder, offsetof(PyThreadState, current_frame))); -#else - // (PyThreadState.cframe|PyThreadState).current_frame = &cur_frame - as_->str( - scratch, - arch::ptr_offset(frame_holder, offsetof(_PyCFrame, current_frame))); -#endif - env_.addAnnotation( - "Set _PyInterpreterFrame as topmost frame", update_linkage_cursor); - - if (tstate_offset == -1) { - as_->ldr(scratch, a64::ptr_post(a64::sp, 16)); - } else { - preserver.remap(); - } -#else - CINDER_UNSUPPORTED -#endif -#else - throw std::runtime_error{ - "linkLightWeightFunctionFrame: Lightweight frames are not supported"}; -#endif -} - -void FrameAsm::linkNormalFunctionFrame( - RegisterPreserver& preserver, - const arch::Gp&, - const arch::Gp& tstate_reg) { - preserver.preserve(); - -#if defined(CINDER_X86_64) - if (kPyDebug) { - as_->mov(x86::rsi, reinterpret_cast(GetFunction()->code.get())); - as_->call( - reinterpret_cast( - JITRT_AllocateAndLinkInterpreterFrame_Debug)); - } else { - as_->call( - reinterpret_cast( - JITRT_AllocateAndLinkInterpreterFrame_Release)); - } - - as_->mov(tstate_reg, x86::rax); -#elif defined(CINDER_AARCH64) - if (kPyDebug) { - as_->mov(a64::x1, reinterpret_cast(GetFunction()->code.get())); - as_->mov(arch::reg_scratch_br, JITRT_AllocateAndLinkInterpreterFrame_Debug); - } else { - as_->mov( - arch::reg_scratch_br, JITRT_AllocateAndLinkInterpreterFrame_Release); - } - - as_->blr(arch::reg_scratch_br); - as_->mov(tstate_reg, a64::x0); -#else - CINDER_UNSUPPORTED -#endif - - preserver.restore(); -} - -void FrameAsm::linkNormalFrame( - RegisterPreserver& preserver, - const arch::Gp& func_reg, - const arch::Gp& tstate_reg) { -#if defined(CINDER_X86_64) - JIT_DCHECK(func_reg == x86::rdi, "func_reg must be rdi"); -#elif defined(CINDER_AARCH64) - JIT_DCHECK(func_reg == a64::x0, "func_reg must be x0"); -#else - CINDER_UNSUPPORTED -#endif - - if (isGen()) { - linkNormalGeneratorFrame(preserver, func_reg, tstate_reg); - } else if (getConfig().frame_mode == FrameMode::kLightweight) { - linkLightWeightFunctionFrame(preserver, func_reg, tstate_reg); - } else { - linkNormalFunctionFrame(preserver, func_reg, tstate_reg); - } -} - -#else - -// Links a normal frame and initializes tstate variable. -void FrameAsm::linkNormalFrame( - RegisterPreserver& preserver, - const arch::Gp& func_reg, - const arch::Gp& tstate_reg) { - preserver.preserve(); - -#if defined(CINDER_X86_64) - as_->mov( - x86::rdi, - reinterpret_cast(codeRuntime()->frameState()->code().get())); - as_->mov( - x86::rsi, - reinterpret_cast( - codeRuntime()->frameState()->builtins().get())); - as_->mov( - x86::rdx, - reinterpret_cast(codeRuntime()->frameState()->globals().get())); - - as_->call(reinterpret_cast(JITRT_AllocateAndLinkFrame)); - as_->mov(tstate_reg, x86::rax); -#elif defined(CINDER_AARCH64) - as_->mov( - a64::x0, - reinterpret_cast(codeRuntime()->frameState()->code().get())); - as_->mov( - a64::x1, - reinterpret_cast( - codeRuntime()->frameState()->builtins().get())); - as_->mov( - a64::x2, - reinterpret_cast(codeRuntime()->frameState()->globals().get())); - - as_->mov(arch::reg_scratch_br, JITRT_AllocateAndLinkFrame); - as_->blr(arch::reg_scratch_br); - as_->mov(tstate_reg, a64::x0); -#else - CINDER_UNSUPPORTED -#endif - - preserver.restore(); -} - -#endif - -#if PY_VERSION_HEX < 0x030C0000 -void FrameAsm::loadTState(const arch::Gp& dst_reg) { -#if defined(CINDER_X86_64) - uint64_t tstate = - reinterpret_cast(&_PyRuntime.gilstate.tstate_current); - - if (fitsSignedInt<32>(tstate)) { - as_->mov(dst_reg, x86::ptr(tstate)); - } else { - as_->mov(dst_reg, tstate); - as_->mov(dst_reg, x86::ptr(dst_reg)); - } -#elif defined(CINDER_AARCH64) - uint64_t tstate = - reinterpret_cast(&_PyRuntime.gilstate.tstate_current); - - as_->mov(dst_reg, tstate); - as_->ldr(dst_reg, a64::ptr(dst_reg)); -#else - CINDER_UNSUPPORTED -#endif -} - -void FrameAsm::generateLinkFrame( - const arch::Gp& func_reg, - const arch::Gp& tstate_reg, - const std::vector>& - save_regs) { - RegisterPreserver preserver(as_, save_regs); - - auto load_tstate_and_move = [&]() { - loadTState(tstate_reg); - preserver.remap(); - }; - - // Prior to 3.12 we did not link a frame on initial generator entry. - if (isGen()) { - load_tstate_and_move(); - return; - } - - switch (GetFunction()->frameMode) { - case FrameMode::kShadow: - load_tstate_and_move(); - break; - case FrameMode::kNormal: - linkNormalFrame(preserver, func_reg, tstate_reg); - break; - case FrameMode::kLightweight: - JIT_ABORT("Lightweight frames are not supported in 3.10"); - break; - } -} - -#else - -void FrameAsm::generateLinkFrame( - const arch::Gp& func_reg, - const arch::Gp& tstate_reg, - const std::vector>& - save_regs) { - JIT_CHECK( - GetFunction()->frameMode != FrameMode::kShadow, - "3.12 doesn't have shadow frames"); - - RegisterPreserver preserver(as_, save_regs); - - linkNormalFrame(preserver, func_reg, tstate_reg); -} -#endif - -void FrameAsm::generateUnlinkFrame([[maybe_unused]] bool is_generator) { -#if defined(CINDER_X86_64) -#ifdef ENABLE_SHADOW_FRAMES - // Unlink shadow frame? The send implementation handles unlinking these for - // generators. - as_->mov(x86::rdi, is_generator ? 0 : 1); - auto saved_rax_ptr = x86::ptr(x86::rbp, -8); -#else - auto saved_rax_ptr = x86::ptr(x86::rbp, -frameHeaderSize()); -#endif - - hir::Type ret_type = func_->return_type; - if (ret_type <= TCDouble) { - as_->movsd(saved_rax_ptr, x86::xmm0); - } else { - as_->mov(saved_rax_ptr, x86::rax); - } - as_->call(reinterpret_cast(JITRT_UnlinkFrame)); - if (ret_type <= TCDouble) { - as_->movsd(x86::xmm0, saved_rax_ptr); - } else { - as_->mov(x86::rax, saved_rax_ptr); - } -#elif defined(CINDER_AARCH64) -#ifdef ENABLE_SHADOW_FRAMES - CINDER_UNSUPPORTED -#else - auto saved_x0_ptr = - arch::ptr_resolve(as_, arch::fp, -frameHeaderSize(), arch::reg_scratch_0); - - hir::Type ret_type = func_->return_type; - if (ret_type <= TCDouble) { - as_->str(a64::d0, saved_x0_ptr); - } else { - as_->str(a64::x0, saved_x0_ptr); - } - as_->mov(arch::reg_scratch_br, JITRT_UnlinkFrame); - as_->blr(arch::reg_scratch_br); - - // It is possible that the scratch register used to compute the pointer was - // clobbered by the call. If so, we need to reload it. This only happens if - // the scratch register is caller-saved, which unfortunately it currently is. - saved_x0_ptr = - arch::ptr_resolve(as_, arch::fp, -frameHeaderSize(), arch::reg_scratch_0); - - if (ret_type <= TCDouble) { - as_->ldr(a64::d0, saved_x0_ptr); - } else { - as_->ldr(a64::x0, saved_x0_ptr); - } -#endif -#else - CINDER_UNSUPPORTED -#endif -} - -#ifdef ENABLE_SHADOW_FRAMES -void FrameAsm::linkOnStackShadowFrame( - const arch::Gp& tstate_reg, - const arch::Gp& scratch_reg) { -#if defined(CINDER_X86_64) - const hir::Function* func = GetFunction(); - FrameMode frame_mode = func->frameMode; - using namespace shadow_frame; - x86::Mem shadow_stack_top_ptr = getStackTopPtr(tstate_reg); - uintptr_t data = - _PyShadowFrame_MakeData(env_.code_rt, PYSF_CODE_RT, PYSF_JIT); - // Save old top of shadow stack - as_->mov(scratch_reg, shadow_stack_top_ptr); - as_->mov(kInFramePrevPtr, scratch_reg); - // Set data - if (frame_mode == FrameMode::kNormal) { - as_->mov(scratch_reg, x86::ptr(tstate_reg, offsetof(PyThreadState, frame))); - static_assert( - PYSF_PYFRAME == 1 && _PyShadowFrame_NumPtrKindBits == 2, - "Unexpected constant"); - as_->bts(scratch_reg, 0); - } else { - as_->mov(scratch_reg, data); - } - as_->mov(kInFrameDataPtr, scratch_reg); - // Set orig_data - // This is only necessary when in normal-frame mode because the frame is - // already materialized on function entry. It is lazily filled when the frame - // is materialized in shadow-frame mode. - if (frame_mode == FrameMode::kNormal) { - as_->mov(scratch_reg, data); - as_->mov(shadow_frame::kInFrameOrigDataPtr, scratch_reg); - } - // Set our shadow frame as top of shadow stack - as_->lea(scratch_reg, kFramePtr); - as_->mov(shadow_stack_top_ptr, scratch_reg); -#else - CINDER_UNSUPPORTED -#endif -} - -void FrameAsm::initializeFrameHeader( - arch::Gp tstate_reg, - arch::Gp scratch_reg) { -#if defined(CINDER_X86_64) - if (!isGen()) { - as_->push(scratch_reg); - linkOnStackShadowFrame(tstate_reg, scratch_reg); - as_->pop(scratch_reg); - } -#else - CINDER_UNSUPPORTED -#endif -} -#endif - -int FrameAsm::frameHeaderSizeExcludingSpillSpace() const { - return jit::frameHeaderSize(func_->code); -} - -int FrameAsm::frameHeaderSize() { -#if defined(ENABLE_SHADOW_FRAMES) - return frameHeaderSizeExcludingSpillSpace(); -#else - return frameHeaderSizeExcludingSpillSpace() + sizeof(void*); -#endif -} - -} // namespace jit::codegen diff --git a/cinderx/Jit/codegen/frame_asm.h b/cinderx/Jit/codegen/frame_asm.h deleted file mode 100644 index e7a925a40..000000000 --- a/cinderx/Jit/codegen/frame_asm.h +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. - -#pragma once - -#include "cinderx/Jit/codegen/arch.h" -#include "cinderx/Jit/codegen/environ.h" -#include "cinderx/Jit/codegen/register_preserver.h" -#include "cinderx/Jit/hir/function.h" -#include "cinderx/Jit/hir/hir.h" - -#include - -#include - -namespace jit::codegen { - -class FrameAsm { - public: - FrameAsm(const hir::Function* func, Environ& env) : func_(func), env_(env) {} - - void initializeFrameHeader(arch::Gp tstate_reg, arch::Gp scratch_reg); - - // Generates the code to link the Python stack frame in. This will ensure - // that the save_regs get transformed from the first of the pair to the - // second of the pair. It will also initialize thread state and leave - // it in tstate. - void generateLinkFrame( - const arch::Gp& func_reg, - const arch::Gp& tstate_reg, - const std::vector>& - save_regs); - - void generateUnlinkFrame(bool is_generator); - - void setAssembler(arch::Builder* as) { - as_ = as; - } - - int frameHeaderSize(); - - private: - const hir::Function* GetFunction() const { - return func_; - } - - CodeRuntime* codeRuntime() const { - return env_.code_rt; - } - - bool isGen() const { - return func_->code->co_flags & kCoFlagsAnyGenerator; - } - -#if defined(CINDER_X86_64) - void incRef( - const arch::Gp& reg, - const arch::Gp& scratch_reg, - const arch::Gp& tstate_reg); -#elif defined(CINDER_AARCH64) - void incRef( - const arch::Gp& reg, - const arch::Gp& scratch_reg, - const arch::Gp& scratch_reg2, - const arch::Gp& tstate_reg); -#else - CINDER_UNSUPPORTED -#endif - -#if defined(CINDER_X86_64) - bool storeConst( - const arch::Gp& reg, - int32_t offset, - void* val, - const arch::Gp& scratch); -#elif defined(CINDER_AARCH64) - bool storeConst( - arch::Builder* as, - const arch::Gp& reg, - int32_t offset, - void* val, - const arch::Gp& scratch0, - const arch::Gp& scratch1); -#else - CINDER_UNSUPPORTED -#endif - - void loadTState(const arch::Gp& dst_reg); - void linkNormalGeneratorFrame( - RegisterPreserver& preserver, - const arch::Gp& func_reg, - const arch::Gp& tstate_reg); - void linkLightWeightFunctionFrame( - RegisterPreserver& preserver, - const arch::Gp& func_reg, - const arch::Gp& tstate_reg); - void linkNormalFunctionFrame( - RegisterPreserver& preserver, - const arch::Gp& func_reg, - const arch::Gp& tstate_reg); - void linkNormalFrame( - RegisterPreserver& preserver, - const arch::Gp& func_reg, - const arch::Gp& tstate_reg); - void linkOnStackShadowFrame( - const arch::Gp& tstate_reg, - const arch::Gp& scratch_reg); - - arch::Builder* as_{}; - const hir::Function* func_; - Environ& env_; - - int frameHeaderSizeExcludingSpillSpace() const; -}; -} // namespace jit::codegen diff --git a/cinderx/Jit/codegen/gen_asm.cpp b/cinderx/Jit/codegen/gen_asm.cpp index b07f611ec..e10dbe228 100644 --- a/cinderx/Jit/codegen/gen_asm.cpp +++ b/cinderx/Jit/codegen/gen_asm.cpp @@ -2,26 +2,20 @@ #include "cinderx/Jit/codegen/gen_asm.h" -#include "internal/pycore_pystate.h" - -#if PY_VERSION_HEX < 0x030C0000 -#include "cinder/exports.h" -#include "internal/pycore_shadow_frame.h" -#endif - -#if PY_VERSION_HEX >= 0x030C0000 #include "internal/pycore_ceval.h" -#endif +#include "internal/pycore_pystate.h" #include "cinderx/Common/extra-py-flags.h" #include "cinderx/Common/log.h" #include "cinderx/Common/py-portability.h" #include "cinderx/Common/util.h" #include "cinderx/Interpreter/interpreter.h" +#include "cinderx/Jit/bytecode.h" #include "cinderx/Jit/codegen/arch.h" #include "cinderx/Jit/codegen/autogen.h" #include "cinderx/Jit/codegen/code_section.h" #include "cinderx/Jit/codegen/gen_asm_utils.h" +#include "cinderx/Jit/compilation_lock.h" #include "cinderx/Jit/compiled_function.h" #include "cinderx/Jit/config.h" #include "cinderx/Jit/context.h" @@ -35,10 +29,13 @@ #include "cinderx/Jit/jit_rt.h" #include "cinderx/Jit/lir/dce.h" #include "cinderx/Jit/lir/generator.h" +#include "cinderx/Jit/lir/linear_scan.h" #include "cinderx/Jit/lir/postalloc.h" #include "cinderx/Jit/lir/postgen.h" #include "cinderx/Jit/lir/printer.h" #include "cinderx/Jit/lir/regalloc.h" +#include "cinderx/Jit/lir/spill_alloc.h" +#include "cinderx/Jit/lir/target_select.h" #include "cinderx/Jit/lir/verify.h" #include "cinderx/Jit/perf_jitdump.h" #include "cinderx/UpstreamBorrow/borrowed.h" @@ -49,19 +46,27 @@ #include #include #include +#include #include #include using namespace asmjit; -using namespace jit; -using namespace jit::hir; -using namespace jit::lir; -using namespace jit::util; +using namespace cinderx::jit; +using namespace cinderx::jit::hir; +using namespace cinderx::jit::lir; +using namespace cinderx::jit::util; -namespace jit::codegen { +namespace cinderx::jit::codegen { namespace { +// prepareForDeopt packs both the reified frame pointer and the +// is_instrumentation_deopt flag into a single uintptr_t returned in RAX/X0. +// The flag is encoded in bit 0 (frame pointers are always >= 8-byte aligned). +// This avoids returning a multi-field struct whose Windows x64 ABI +// hidden-pointer return semantics would conflict with the hand-rolled deopt +// trampoline. Callers unpack with: frame = result & ~1, flag = result & 1. + #define ASM_CHECK_THROW(exp) \ { \ auto err = (exp); \ @@ -84,32 +89,40 @@ namespace { // Scratch register used by the various deopt trampolines. [[maybe_unused]] const auto deopt_scratch_reg = arch::reg_scratch_deopt; -// Set the frame pointer to "original frame pointer" value when called in the -// context of a generator. -void RestoreOriginalGeneratorFramePointer(arch::Builder* as) { -#if defined(CINDER_X86_64) - size_t original_frame_pointer_offset = - offsetof(GenDataFooter, originalFramePointer); - as->mov(x86::rbp, x86::ptr(x86::rbp, original_frame_pointer_offset)); -#elif defined(CINDER_AARCH64) - size_t original_frame_pointer_offset = - offsetof(GenDataFooter, originalFramePointer); - as->ldr( - arch::fp, - arch::ptr_resolve( - as, arch::fp, original_frame_pointer_offset, arch::reg_scratch_0)); -#else - CINDER_UNSUPPORTED -#endif +// Raise an exception if an LIR function exceeds a reasonable size. +void checkLirSize(const lir::Function& func) { + auto name = [&] { + auto hir_func = func.hirFunc(); + return hir_func != nullptr ? hir_func->fullname : ""; + }; + + auto num_blocks = func.getNumBasicBlocks(); + auto max_blocks = getConfig().max_lir_blocks; + if (num_blocks > max_blocks) { + throw std::runtime_error{fmt::format( + "LIR function '{}' has too many basic blocks ({}, max={})", + name(), + num_blocks, + max_blocks)}; + } + + auto num_instrs = func.getNumInstrs(); + auto max_instrs = getConfig().max_lir_instrs; + if (num_instrs > max_instrs) { + throw std::runtime_error{fmt::format( + "LIR function '{}' has too many instructions ({}, max={})", + name(), + num_instrs, + max_instrs)}; + } } void raiseUnboundLocalError(BorrowedRef<> name) { // name is converted into a `char*` in format_exc_check_arg - const char* msg = PY_VERSION_HEX >= 0x030C0000 - ? "cannot access local variable '%.200s' where it is not associated with " - "a value" - : "local variable '%.200s' referenced before assignment"; + const char* msg = + "cannot access local variable '%.200s' where it is not associated with " + "a value"; _PyEval_FormatExcCheckArg( _PyThreadState_GET(), PyExc_UnboundLocalError, msg, name); @@ -118,11 +131,9 @@ void raiseUnboundLocalError(BorrowedRef<> name) { void raiseUnboundFreevarError(BorrowedRef<> name) { // name is converted into a `char*` in format_exc_check_arg - const char* msg = PY_VERSION_HEX >= 0x030C0000 - ? "cannot access free variable '%.200s' where it is not associated with a" - " value in enclosing scope" - : "free variable '%.200s' referenced before assignment in enclosing " - "scope"; + const char* msg = + "cannot access free variable '%.200s' where it is not associated with a" + " value in enclosing scope"; _PyEval_FormatExcCheckArg(_PyThreadState_GET(), PyExc_NameError, msg, name); } @@ -135,8 +146,6 @@ void raiseAttributeError(BorrowedRef<> receiver, BorrowedRef<> name) { name); } -#if PY_VERSION_HEX >= 0x030C0000 - // Helper to recursively reify the lightweight frames. We need to reify the // outermost lightweight frame first and work inwards to have the frames // allocated correctly on the slab. We then need to update the inner functions @@ -168,70 +177,96 @@ _PyInterpreterFrame* reifyLightweightFrames( return cur_frame; } -#endif - -CiPyFrameObjType* prepareForDeopt( +uintptr_t prepareForDeopt( const uint64_t* regs, CodeRuntime* code_runtime, std::size_t deopt_idx) { JIT_CHECK(deopt_idx != -1ull, "deopt_idx must be valid"); const DeoptMetadata& deopt_meta = code_runtime->getDeoptMetadata(deopt_idx); PyThreadState* tstate = _PyThreadState_UncheckedGet(); -#if PY_VERSION_HEX < 0x030C0000 - Ref f = materializePyFrameForDeopt(tstate); - - PyFrameObject* frame = f.release(); - PyFrameObject* frame_iter = frame; - _PyShadowFrame* sf_iter = tstate->shadow_frame; - // Iterate one past the inline depth because that is the caller frame. - for (int i = deopt_meta.inline_depth(); i >= 0; i--) { - // Transfer ownership of shadow frame to the interpreter. The associated - // Python frame will be ignored during future attempts to materialize the - // stack. - _PyShadowFrame_SetOwner(sf_iter, PYSF_INTERP); - reifyFrame(frame_iter, deopt_meta, deopt_meta.frame_meta.at(i), regs); - frame_iter = frame_iter->f_back; - sf_iter = sf_iter->prev; - } -#else + bool is_instrumentation_deopt = false; _PyInterpreterFrame* frame = interpFrameFromThreadState(tstate); - if (getConfig().frame_mode == FrameMode::kLightweight) { - frame = reifyLightweightFrames( - tstate, deopt_meta, deopt_meta.inline_depth(), frame); - if (frame == nullptr) { - Py_FatalError("Cannot recover from OOM"); + // Check JIT_FRAME_DEOPT_PATCHED on the outermost frame's header before + // reification destroys it. Walk past inlined frames to find the outer one. +#ifdef ENABLE_LIGHTWEIGHT_FRAMES + { + _PyInterpreterFrame* outer = frame; + for (size_t i = 0; i < deopt_meta.inline_depth(); i++) { + outer = outer->previous; } - setCurrentFrame(tstate, frame); + is_instrumentation_deopt = + (jitFrameGetHeader(outer)->frame_status & JIT_FRAME_DEOPT_PATCHED) != 0; } +#endif + + frame = reifyLightweightFrames( + tstate, deopt_meta, deopt_meta.inline_depth(), frame); + if (frame == nullptr) { + Py_FatalError("Cannot recover from OOM"); + } + setCurrentFrame(tstate, frame); _PyInterpreterFrame* frame_iter = frame; + // Shared by every frame of this deopt so that a live value referenced by + // more than one of them is only boxed once. + MemoryView mem{regs}; + // Iterate one past the inline depth because that is the caller frame. for (int i = deopt_meta.inline_depth(); i >= 0; i--) { // Transfer ownership of a light weight frame to the interpreter. The // associated Python frame will be ignored during future attempts to // materialize the stack. - reifyFrame(frame_iter, deopt_meta, deopt_meta.frame_meta.at(i), regs); + reifyFrame( + frame_iter, + deopt_meta, + deopt_meta.frame_meta.at(i), + mem, + is_instrumentation_deopt); frame_iter = frame_iter->previous; } + // For instrumentation deopts where the bytecode's C call completed + // (reason != kPeriodicTaskFailure), push its return value onto the + // operand stack on top of the pre-instruction state restored by reifyStack. + if (is_instrumentation_deopt) { + if (deopt_meta.reason != DeoptReason::kPeriodicTaskFailure && + !PyErr_Occurred()) { + PyObject* retval = reinterpret_cast( + regs[codegen::arch::reg_general_return_loc.loc]); + if (retval != nullptr) { +#if PY_VERSION_HEX >= 0x030E0000 + *(frame->stackpointer) = PyStackRef_FromPyObjectSteal(retval); + frame->stackpointer++; +#else + frame->localsplus[frame->stacktop] = Ci_STACK_STEAL(retval); + frame->stacktop++; #endif + } else { + PyErr_SetString( + PyExc_SystemError, + "JIT instrumentation deopt: call returned NULL without " + "setting an exception"); + } + } + } // Clear our references now that we've transferred them to the frame - MemoryView mem{regs}; - Ref<> deopt_obj = profileDeopt(deopt_meta, mem); + Ref<> deopt_obj; + if (!is_instrumentation_deopt) { + // TODO(T262342844): Add USDT support for instrumentation-related deopts. + deopt_obj = profileDeopt(deopt_meta, mem); + } auto ctx = getContext(); ctx->recordDeopt(code_runtime, deopt_idx, deopt_obj); releaseRefs(deopt_meta, mem); -#if PY_VERSION_HEX >= 0x030C0000 if (_PyFrame_GetCode(frame)->co_flags & kCoFlagsAnyGenerator) { BorrowedRef base_gen = _PyGen_GetGeneratorFromFrame(frame); JitGenObject* gen = JitGenObject::cast(base_gen.get()); JIT_CHECK(gen != nullptr, "Not a JIT generator"); deopt_jit_gen_object_only(gen); } -#endif - if (!PyErr_Occurred()) { + if (!PyErr_Occurred() && !is_instrumentation_deopt) { auto reason = deopt_meta.reason; switch (reason) { case DeoptReason::kGuardFailure: { @@ -252,86 +287,50 @@ CiPyFrameObjType* prepareForDeopt( raiseUnboundFreevarError(deopt_meta.eh_name); break; case DeoptReason::kUnhandledException: + case DeoptReason::kPeriodicTaskFailure: JIT_ABORT("unhandled exception without error set"); case DeoptReason::kRaiseStatic: JIT_ABORT("Lost exception when raising static exception"); } } - return frame; + // Pack the frame pointer and the instrumentation-deopt flag into a single + // register-width value. Bit 0 carries the flag; the remaining bits carry + // the pointer (which is always at least 8-byte aligned, so bit 0 is free). + return reinterpret_cast(frame) | + static_cast(is_instrumentation_deopt); } -#if PY_VERSION_HEX < 0x030C0000 -PyObject* resumeInInterpreter( - PyFrameObject* frame, - CodeRuntime* code_runtime, - std::size_t deopt_idx) { - if (frame->f_gen) { - auto gen = reinterpret_cast(frame->f_gen); - // It's safe to call jitgen_data_free directly here, rather than - // through _PyJIT_GenDealloc. Ownership of all references have been - // transferred to the frame. - jitgen_data_free(gen); - } - PyThreadState* tstate = PyThreadState_Get(); - PyObject* result = nullptr; - // Resume all of the inlined frames and the caller - const DeoptMetadata& deopt_meta = code_runtime->getDeoptMetadata(deopt_idx); - int inline_depth = deopt_meta.inline_depth(); - int err_occurred = - (deopt_meta.reason != DeoptReason::kGuardFailure && - deopt_meta.reason != DeoptReason::kRaise); - while (inline_depth >= 0) { - // Consider skipping resuming frames that do not have try/catch. Will - // require re-adding _PyShadowFrame_Pop back for non-generators and - // unlinking the frame manually. - - // We need to maintain the invariant that there is at most one shadow frame - // on the shadow stack for each frame on the Python stack. Unless we are a - // a generator, the interpreter will insert a new entry on the shadow stack - // when execution resumes there, so we remove our entry. - if (!frame->f_gen) { - _PyShadowFrame_Pop(tstate, tstate->shadow_frame); - } - // Resume one frame. - PyFrameObject* prev_frame = frame->f_back; - // Delegate management of `tstate->frame` to the interpreter loop. On - // entry, it expects that tstate->frame points to the frame for the calling - // function. - JIT_CHECK(tstate->frame == frame, "unexpected frame at top of stack"); - tstate->frame = prev_frame; - result = PyEval_EvalFrameEx(frame, err_occurred); - JITRT_DecrefFrame(frame); - frame = prev_frame; - - err_occurred = result == nullptr; - // Push the previous frame's result onto the value stack. We can't push - // after resuming because f_stacktop is nullptr during execution of a frame. - if (!err_occurred) { - if (inline_depth > 0) { - // The caller is at inline depth 0, so we only attempt to push the - // result onto the stack in the deeper (> 0) frames. Otherwise, we - // should just return the value from the native code in the way our - // native calling convention requires. - frame->f_valuestack[frame->f_stackdepth++] = result; +// Set up f_trace/f_trace_lines for sys.settrace compatibility on deopted +// frames. CPython normally sets these during the RESUME opcode at function +// entry, but deopted frames resume mid-function and skip RESUME. +void setupTraceForDeoptedFrame( + _PyInterpreterFrame* frame, + PyThreadState* tstate) { + if (tstate->c_tracefunc != nullptr && + frame->owner != FRAME_OWNED_BY_GENERATOR) { + PyFrameObject* fobj = _PyFrame_GetFrameObject(frame); + if (fobj != nullptr) { + fobj->f_trace_lines = 1; + if (fobj->f_trace == nullptr && tstate->c_traceobj != nullptr) { + fobj->f_trace = Py_NewRef(tstate->c_traceobj); } } - inline_depth--; } - return result; } -#else - PyObject* resumeInInterpreter( _PyInterpreterFrame* frame, CodeRuntime* code_runtime, - std::size_t deopt_idx) { + std::size_t deopt_idx, + bool is_instrumentation_deopt) { JIT_CHECK(code_runtime != nullptr, "CodeRuntime cannot be a nullptr"); PyThreadState* tstate = PyThreadState_Get(); const DeoptMetadata& deopt_meta = code_runtime->getDeoptMetadata(deopt_idx); - int err_occurred = shouldResumeInterpreterInErrorHandler(deopt_meta.reason); + + // For instrumentation deopts, only enter error handler if actually excepted. + int err_occurred = PyErr_Occurred() != nullptr; PyObject* result = nullptr; // Resume all of the inlined frames and the caller @@ -368,16 +367,36 @@ PyObject* resumeInInterpreter( // exception state, so we don't need to do any cleanup after // _PyEval_EvalFrame. Note: We only set this up if it's not already set // (e.g., jitgen_am_send may have already set it up before we got here). + // + // Additionally, if the generator was never returned to the caller (i.e., + // exception occurred before RETURN_GENERATOR), we need to decref the + // generator since nobody owns the reference. We detect this by checking + // if gi_frame_state was FRAME_CREATED before executing. + PyGenObject* gen_to_cleanup = nullptr; if (frame->owner == FRAME_OWNED_BY_GENERATOR) { PyGenObject* gen = _PyGen_GetGeneratorFromFrame(frame); + if (gen->gi_frame_state == FRAME_CREATED) { + // This generator was never returned to the caller (before + // RETURN_GENERATOR). If an exception occurs, we need to clean it up. + gen_to_cleanup = gen; + } if (tstate->exc_info != &gen->gi_exc_state) { gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; } } + setupTraceForDeoptedFrame(frame, tstate); + result = _PyEval_EvalFrame(tstate, frame, err_occurred); + // If exception occurred before RETURN_GENERATOR, the generator was never + // returned to anyone. The JIT created the generator early, but the caller + // never received it. We need to decref it to avoid a memory leak. + if (result == nullptr && gen_to_cleanup != nullptr) { + Py_DECREF(gen_to_cleanup); + } + frame = prev_frame; err_occurred = result == nullptr; @@ -399,8 +418,6 @@ PyObject* resumeInInterpreter( return result; } -#endif - void* finalizeCode(arch::Builder& builder, std::string_view name) { if (auto err = builder.finalize(); err != kErrorOk) { throw std::runtime_error{fmt::format( @@ -409,7 +426,8 @@ void* finalizeCode(arch::Builder& builder, std::string_view name) { DebugUtils::errorAsString(err))}; } - ICodeAllocator* code_allocator = cinderx::getModuleState()->codeAllocator(); + ICodeAllocator* code_allocator = + cinderx::getModuleState()->code_allocator.get(); AllocateResult result = code_allocator->addCode(builder.code()); if (result.error != kErrorOk) { throw std::runtime_error{fmt::format( @@ -421,445 +439,96 @@ void* finalizeCode(arch::Builder& builder, std::string_view name) { return result.addr; } -// Generate the final stage trampoline that is responsible for finishing -// execution in the interpreter and then returning the result to the caller. -void* generateDeoptTrampoline(bool generator_mode) { - auto mod_state = cinderx::getModuleState(); - if (mod_state == nullptr) { - throw std::runtime_error{ - "CinderX not initialized, cannot generate deopt trampolines"}; - } +// Emit machine code from LIR blocks by translating each instruction via the +// AutoTranslator. Populates env->block_label_map and records annotations. +// +// When |code| and |metadata| are non-null, CodeSectionOverride is applied per +// block (for multi-section support in normal JIT functions). When they are +// null the section override is skipped (standalone trampolines). +void emitLIRBlocks( + Environ* env, + lir::Function* lir_func, + const asmjit::CodeHolder* code = nullptr, + CodeHolderMetadata* metadata = nullptr) { + auto* as = env->as; + auto& blocks = lir_func->basicBlocks(); - auto name = - generator_mode ? "deopt_trampoline_generators" : "deopt_trampoline"; + for (auto& basicblock : blocks) { + env->block_label_map.emplace(basicblock, as->newLabel()); + } - CodeHolder code; - ICodeAllocator* code_allocator = mod_state->codeAllocator(); - ASM_CHECK(code.init(code_allocator->asmJitEnvironment()), name); - arch::Builder a(&code); - Annotations annot; + std::string pending_annotation; + asmjit::BaseNode* annotation_cursor = nullptr; -#if defined(CINDER_X86_64) - auto annot_cursor = a.cursor(); - // When we get here the stack has the following layout. The space on the - // stack for the call arg buffer / LOAD_METHOD scratch space is always safe - // to read, but its contents will depend on the function being compiled as - // well as the program point at which deopt occurs. We pass a pointer to it - // into the frame reification code so that it can properly reconstruct the - // interpreter's stack when the the result of a LOAD_METHOD is on the - // stack. See the comments in reifyStack in deopt.cpp for more details. - // - // +-------------------------+ - // | ... | - // | ? call arg buffer | - // | ^ LOAD_METHOD scratch | - // +-------------------------+ <-- end of JIT's fixed frame - // | index of deopt metadata | - // | saved rip | - // | padding | - // | padding | - // | address of CodeRuntime | - // | address of epilogue | - // | r15 | <-- rsp - // +-------------------------+ - // - // Save registers for use in frame reification. Once these are saved we're - // free to clobber any caller-saved registers. - // - // IF YOU USE CALLEE-SAVED REGISTERS YOU HAVE TO RESTORE THEM MANUALLY BEFORE - // THE EXITING THE TRAMPOLINE. - a.push(x86::r14); - a.push(x86::r13); - a.push(x86::r12); - a.push(x86::r11); - a.push(x86::r10); - a.push(x86::r9); - a.push(x86::r8); - a.push(x86::rdi); - a.push(x86::rsi); - a.push(x86::rbp); - a.push(x86::rsp); - a.push(x86::rbx); - a.push(x86::rdx); - a.push(x86::rcx); - a.push(x86::rax); - - if (generator_mode) { - // Restore the original frame pointer for use in epilogue. - RestoreOriginalGeneratorFramePointer(&a); - } + for (lir::BasicBlock* basicblock : blocks) { + // Optional section override for multi-section code layout. + std::optional section_override; + if (code != nullptr && metadata != nullptr) { + section_override.emplace(as, code, metadata, basicblock->section()); + } - annot.add("Save registers", &a, annot_cursor); + as->bind(env->block_label_map[basicblock]); + for (auto& instr : basicblock->instructions()) { + asmjit::BaseNode* cursor = as->cursor(); - // Set up a stack frame for the trampoline so that: - // - // 1. Runtime code in the JIT that is used to update PyFrameObjects can find - // the saved rip at the expected location immediately following the end of - // the JIT's fixed frame. See getIP(). - // - // 2. The JIT-compiled function shows up in C stack straces when it is - // deopting. Only the deopt trampoline will appear in the trace if - // we don't open a frame. - // - // Right now the stack has the following layout: - // - // +-------------------------+ <-- end of JIT's fixed frame - // | index of deopt metadata | - // | saved rip | - // | padding | - // | padding | - // | address of CodeRuntime | - // | address of epilogue | - // | r15 | - // | ... | - // | rax | <-- rsp - // +-------------------------+ - // - // We want our frame to look like: - // - // +-------------------------+ <-- end of JIT's fixed frame - // | saved rip | - // | saved rbp | <-- rbp - // | padding | - // | index of deopt metadata | - // | address of CodeRuntime | - // | address of epilogue | - // | r15 | - // | ... | - // | rax | <-- rsp - // +-------------------------+ - - annot_cursor = a.cursor(); - - // Setting up first argument to prepareForDeopt, the address of the saved - // registers. - a.mov(x86::rdi, x86::rsp); - - // Load the saved rip passed to us from the JIT-compiled function, which - // resides where we're supposed to save rbp. - auto saved_rip = x86::rcx; - auto saved_rbp_addr = x86::ptr(x86::rsp, (NUM_GP_REGS + 4) * kPointerSize); - a.mov(saved_rip, saved_rbp_addr); - - // Save rbp and set up our frame. - a.mov(saved_rbp_addr, x86::rbp); - a.lea(x86::rbp, saved_rbp_addr); - - // Load the index of the deopt metadata, which resides where we're supposed to - // save rip. - auto deopt_idx = x86::rdx; - auto saved_rip_addr = x86::ptr(x86::rbp, kPointerSize); - a.mov(deopt_idx, saved_rip_addr); - a.mov(saved_rip_addr, saved_rip); - - // Save the deopt metadata index to the lower padding slot. - auto deopt_idx_addr = x86::ptr(x86::rbp, -2 * kPointerSize); - a.mov(deopt_idx_addr, deopt_idx); - - // Fetch the CodeRuntime address from the stack. - auto code_rt_addr = x86::ptr(x86::rbp, -3 * kPointerSize); - auto code_rt = x86::rsi; - a.mov(code_rt, code_rt_addr); - - annot.add("Shuffle rip, rbp, and deopt index", &a, annot_cursor); - - // Prep the frame for evaluation in the interpreter. - // - // We pass the array of saved registers, a pointer to the code runtime, and - // the index of the deopt metadata. - annot_cursor = a.cursor(); - - static_assert( - std::is_same_v< - decltype(prepareForDeopt), - CiPyFrameObjType*(const uint64_t*, CodeRuntime*, std::size_t)>, - "prepareForDeopt has unexpected signature"); - a.call(reinterpret_cast(prepareForDeopt)); - - // Clean up saved registers. - // - // This isn't strictly necessary but saves 128 bytes on the stack if we end - // up resuming in the interpreter. - a.add(x86::rsp, (NUM_GP_REGS - 1) * kPointerSize); - - // We have to restore our scratch register manually since it's callee-saved - // and the stage 2 trampoline used it to hold the address of this - // trampoline. We can't rely on the JIT epilogue to restore it for us, as the - // JIT-compiled code may not have spilled it. - a.pop(deopt_scratch_reg); - - annot.add("prepareForDeopt", &a, annot_cursor); - - // Resume execution in the interpreter. - annot_cursor = a.cursor(); - - // First argument: frame returned from prepareForDeopt. - a.mov(x86::rdi, x86::rax); - // Second argument: CodeRuntime, restored from the stack after - // prepareForDeopt. - a.mov(code_rt, code_rt_addr); - // Third argument: DeoptMetadata index, restored from the stack after - // prepareForDeopt. - a.mov(deopt_idx, deopt_idx_addr); - static_assert( - std::is_same_v< - decltype(resumeInInterpreter), - PyObject*(CiPyFrameObjType*, CodeRuntime*, std::size_t)>, - "resumeInInterpreter has unexpected signature"); - a.call(reinterpret_cast(resumeInInterpreter)); - - // If we return a primitive and prepareForDeopt returned null, we need that - // null in edx/xmm1 to signal error to our caller. Since this trampoline is - // shared, we do this move unconditionally, but even if not needed, it's - // harmless. (To eliminate it, we'd need another trampoline specifically for - // deopt of primitive-returning functions, just to do this one move.) - a.mov(x86::edx, x86::eax); - a.movq(x86::xmm1, x86::eax); - - annot.add("resumeInInterpreter", &a, annot_cursor); - - // Now we're done. Get the address of the epilogue and jump there. - annot_cursor = a.cursor(); - - auto epilogue_addr = x86::ptr(x86::rbp, -4 * kPointerSize); - a.mov(x86::rdi, epilogue_addr); - // Remove our frame from the stack - a.leave(); - // Clear the saved rip. Normally this would be handled by a `ret`; we must - // clear it manually because we're jumping directly to the epilogue. - a.sub(x86::rsp, -kPointerSize); - a.jmp(x86::rdi); - annot.add("Jump to real epilogue", &a, annot_cursor); + // Check for annotation BEFORE translating so cursor captures the + // position before the instruction's code is emitted. + auto* annot_text = lir_func->getAnnotation(instr.get()); + if (annot_text) { + // Close any previous pending annotation. + if (!pending_annotation.empty()) { + JIT_DCHECK(annotation_cursor != nullptr, "should be set"); + env->addAnnotation(std::move(pending_annotation), annotation_cursor); + } + // Start new annotation range from current cursor position. + pending_annotation = *annot_text; + annotation_cursor = cursor; + } - void* result = finalizeCode(a, name); - JIT_LOGIF( - getConfig().log.dump_asm, - "Disassembly for {}\n{}", - name, - annot.disassemble(result, code)); + env->suppress_annotations = !pending_annotation.empty(); + autogen::AutoTranslator::getInstance().translateInstr(env, instr.get()); + env->suppress_annotations = false; - auto code_size = code.codeSize(); - register_raw_debug_symbol(name, __FILE__, __LINE__, result, code_size, 0); + if (!pending_annotation.empty()) { + // Under an active annotation — don't emit per-instruction annotations. + } else if (instr->origin() != nullptr) { + env->addAnnotation(instr.get(), cursor); + } + } + // Close pending annotation at block boundary. + if (!pending_annotation.empty()) { + env->addAnnotation(std::move(pending_annotation), annotation_cursor); + pending_annotation.clear(); + } + } +} - std::vector> code_sections; - populateCodeSections(code_sections, code, result); - code_sections.emplace_back(result, code_size); - perf::registerFunction(code_sections, name); - return result; -#elif defined(CINDER_AARCH64) - auto annot_cursor = a.cursor(); - // When we get here the stack has the following layout. The space on the - // stack for the call arg buffer / LOAD_METHOD scratch space is always safe - // to read, but its contents will depend on the function being compiled as - // well as the program point at which deopt occurs. We pass a pointer to it - // into the frame reification code so that it can properly reconstruct the - // interpreter's stack when the the result of a LOAD_METHOD is on the - // stack. See the comments in reifyStack in deopt.cpp for more details. - // - // +-------------------------+ - // | ... | - // | ? call arg buffer | - // | ^ LOAD_METHOD scratch | - // +-------------------------+ <-- end of JIT's fixed frame - // | index of deopt metadata | - // | saved pc | - // | padding (8 bytes) | - // | padding (8 bytes) | - // | address of CodeRuntime | - // | address of epilogue | - // | fp | - // | x28 | <-- sp - // +-------------------------+ - // - // Save registers for use in frame reification. Once these are saved we're - // free to clobber any caller-saved registers. - // - // IF YOU USE CALLEE-SAVED REGISTERS YOU HAVE TO RESTORE THEM MANUALLY BEFORE - // THE EXITING THE TRAMPOLINE. - a.stp(a64::x0, a64::x1, a64::ptr_pre(a64::sp, -16 * 14)); - a.stp(a64::x2, a64::x3, a64::ptr(a64::sp, 16 * 1)); - a.stp(a64::x4, a64::x5, a64::ptr(a64::sp, 16 * 2)); - a.stp(a64::x6, a64::x7, a64::ptr(a64::sp, 16 * 3)); - a.stp(a64::x8, a64::x9, a64::ptr(a64::sp, 16 * 4)); - a.stp(a64::x10, a64::x11, a64::ptr(a64::sp, 16 * 5)); - a.stp(a64::x12, a64::x13, a64::ptr(a64::sp, 16 * 6)); - a.stp(a64::x14, a64::x15, a64::ptr(a64::sp, 16 * 7)); - a.stp(a64::x16, a64::x17, a64::ptr(a64::sp, 16 * 8)); - a.stp(a64::x18, a64::x19, a64::ptr(a64::sp, 16 * 9)); - a.stp(a64::x20, a64::x21, a64::ptr(a64::sp, 16 * 10)); - a.stp(a64::x22, a64::x23, a64::ptr(a64::sp, 16 * 11)); - a.stp(a64::x24, a64::x25, a64::ptr(a64::sp, 16 * 12)); - a.stp(a64::x26, a64::x27, a64::ptr(a64::sp, 16 * 13)); - - if (generator_mode) { - // Restore original frame pointer for use in epilogue. - RestoreOriginalGeneratorFramePointer(&a); +// Emit LIR blocks to machine code, finalize, register debug/perf symbols, and +// return the entry address. Shared by all standalone trampoline generators. +static void* emitAndRegisterTrampoline( + lir::Function* lir_func, + const char* name) { + auto mod_state = cinderx::getModuleState(); + if (mod_state == nullptr) { + throw std::runtime_error{ + fmt::format("CinderX not initialized, cannot generate {}", name)}; } - annot.add("Save registers", &a, annot_cursor); + CodeHolder code; + ICodeAllocator* code_allocator = mod_state->code_allocator.get(); + ASM_CHECK(code.init(code_allocator->asmJitEnvironment()), name); + arch::Builder a(&code); - // Set up a stack frame for the trampoline so that: - // - // 1. Runtime code in the JIT that is used to update PyFrameObjects can find - // the saved pc at the expected location immediately following the end of - // the JIT's fixed frame. See getIP(). - // - // 2. The JIT-compiled function shows up in C stack traces when it is - // deopting. Only the deopt trampoline will appear in the trace if - // we don't open a frame. - // - // Right now the stack has the following layout: - // - // +-------------------------+ <-- end of JIT's fixed frame - // | index of deopt metadata | - // | saved pc | - // | padding (8 bytes) | - // | padding (8 bytes) | - // | address of CodeRuntime | - // | address of epilogue | - // | fp | - // | x28 | - // | ... | - // | x0 | <-- sp - // +-------------------------+ - // - // We want our frame to look like: - // - // +-------------------------+ <-- end of JIT's fixed frame - // | saved pc | - // | saved fp | <-- fp - // | padding (8 bytes) | - // | index of deopt metadata | - // | address of CodeRuntime | - // | address of epilogue | - // | fp | - // | x28 | - // | ... | - // | x0 | <-- sp - // +-------------------------+ - - annot_cursor = a.cursor(); - - // Setting up first argument to prepareForDeopt, the address of the saved - // registers. - a.mov(a64::x0, a64::sp); - - // Load the saved pc passed to us from the JIT-compiled function, which - // resides where we're supposed to save the frame pointer. - const int saved_regs_slots = 30; - const int saved_metadata_slots = 4; - - auto saved_pc = a64::x3; - auto saved_fp_offset = - (saved_regs_slots + saved_metadata_slots) * kPointerSize; - a.ldr( - saved_pc, - arch::ptr_resolve(&a, a64::sp, saved_fp_offset, arch::reg_scratch_0)); - - // Save the frame pointer and set up our frame. - a.str( - arch::fp, - arch::ptr_resolve(&a, a64::sp, saved_fp_offset, arch::reg_scratch_0)); - a.add(arch::fp, a64::sp, saved_fp_offset); - - // Load the index of the deopt metadata, which resides where we're supposed to - // save the pc. - auto deopt_idx = a64::x2; - a.ldr( - deopt_idx, - arch::ptr_resolve(&a, arch::fp, kPointerSize, arch::reg_scratch_0)); - a.str( - saved_pc, - arch::ptr_resolve(&a, arch::fp, kPointerSize, arch::reg_scratch_0)); - - // Save the deopt metadata index to the lower padding slot. - auto deopt_idx_addr = - arch::ptr_resolve(&a, arch::fp, -2 * kPointerSize, arch::reg_scratch_0); - a.str(deopt_idx, deopt_idx_addr); - - // Fetch the CodeRuntime address from the stack. - auto code_rt_addr = - arch::ptr_resolve(&a, arch::fp, -3 * kPointerSize, arch::reg_scratch_0); - auto code_rt = a64::x1; - a.ldr(code_rt, code_rt_addr); - - annot.add("Shuffle pc, fp, and deopt index", &a, annot_cursor); - - // Prep the frame for evaluation in the interpreter. - // - // We pass the array of saved registers, a pointer to the code runtime, and - // the index of the deopt metadata. - annot_cursor = a.cursor(); - - static_assert( - std::is_same_v< - decltype(prepareForDeopt), - CiPyFrameObjType*(const uint64_t*, CodeRuntime*, std::size_t)>, - "prepareForDeopt has unexpected signature"); - a.mov(arch::reg_scratch_br, prepareForDeopt); - a.blr(arch::reg_scratch_br); - - // Clean up saved registers. - // - // This isn't strictly necessary but saves 128 bytes on the stack if we end - // up resuming in the interpreter. - a.add(a64::sp, a64::sp, (saved_regs_slots - 2) * kPointerSize); - - // We have to restore our scratch register manually since it's callee-saved - // and the stage 2 trampoline used it to hold the address of this - // trampoline. We can't rely on the JIT epilogue to restore it for us, as the - // JIT-compiled code may not have spilled it. - a.ldr(deopt_scratch_reg, a64::ptr(a64::sp)); - - annot.add("prepareForDeopt", &a, annot_cursor); - - // Resume execution in the interpreter. - annot_cursor = a.cursor(); - - // First argument: frame returned from prepareForDeopt. - // already in x0 - // Second argument: CodeRuntime, restored from the stack after - // prepareForDeopt. - a.ldr(code_rt, code_rt_addr); - // Third argument: DeoptMetadata index, restored from the stack after - // prepareForDeopt. - a.ldr(deopt_idx, deopt_idx_addr); - static_assert( - std::is_same_v< - decltype(resumeInInterpreter), - PyObject*(CiPyFrameObjType*, CodeRuntime*, std::size_t)>, - "resumeInInterpreter has unexpected signature"); - a.mov(arch::reg_scratch_br, resumeInInterpreter); - a.blr(arch::reg_scratch_br); - - // If we return a primitive and prepareForDeopt returned null, we need that - // null in w2/d1 to signal error to our caller. Since this trampoline is - // shared, we do this move unconditionally, but even if not needed, it's - // harmless. (To eliminate it, we'd need another trampoline specifically for - // deopt of primitive-returning functions, just to do this one move.) - a.mov(a64::w2, a64::w0); - a.fmov(a64::d1, a64::x0); - - annot.add("resumeInInterpreter", &a, annot_cursor); - - // Now we're done. Get the address of the epilogue and jump there. - annot_cursor = a.cursor(); - - auto epilogue_addr = - arch::ptr_resolve(&a, arch::fp, -4 * kPointerSize, arch::reg_scratch_0); - a.ldr(arch::reg_scratch_br, epilogue_addr); - // Remove our frame from the stack - a.mov(a64::sp, arch::fp); - a.ldp(arch::fp, arch::lr, a64::ptr_post(a64::sp, 16)); - a.br(arch::reg_scratch_br); - annot.add("Jump to real epilogue", &a, annot_cursor); + Environ env; + env.as = &a; + emitLIRBlocks(&env, lir_func); void* result = finalizeCode(a, name); JIT_LOGIF( getConfig().log.dump_asm, "Disassembly for {}\n{}", name, - annot.disassemble(result, code)); + env.annotations.disassemble(result, code)); auto code_size = code.codeSize(); register_raw_debug_symbol(name, __FILE__, __LINE__, result, code_size, 0); @@ -867,99 +536,109 @@ void* generateDeoptTrampoline(bool generator_mode) { std::vector> code_sections; populateCodeSections(code_sections, code, result); code_sections.emplace_back(result, code_size); +#ifndef WIN32 perf::registerFunction(code_sections, name); - return result; -#else - CINDER_UNSUPPORTED - return nullptr; #endif + return result; +} + +void* generateDeoptTrampoline(bool generator_mode) { + lir::Function lir_func; + lir::GenerateDeoptTrampolineBlocks( + &lir_func, + generator_mode, + reinterpret_cast(prepareForDeopt), + reinterpret_cast(resumeInInterpreter)); + + return emitAndRegisterTrampoline( + &lir_func, + generator_mode ? "deopt_trampoline_generators" : "deopt_trampoline"); } void* generateFailedDeferredCompileTrampoline() { - auto mod_state = cinderx::getModuleState(); - if (mod_state == nullptr) { - throw std::runtime_error{ - "CinderX not initialized, cannot generate deopt trampolines"}; + lir::Function lir_func; + lir::GenerateFailedDeferredCompileBlocks( + &lir_func, reinterpret_cast(rt::failedDeferredCompileShim)); + + return emitAndRegisterTrampoline( + &lir_func, "failedDeferredCompileTrampoline"); +} + +// Helper template implementing double-checked locking for lazy trampoline +// initialization. |slot| is the atomic cache, |generator| is a callable that +// creates the trampoline when it has not yet been initialized. +template +void* getOrCreateTrampoline(std::atomic& slot, Generator&& generator) { + void* trampoline = slot.load(std::memory_order_acquire); + if (trampoline == nullptr) { + JITCompilationLock lock; + trampoline = slot.load(std::memory_order_relaxed); + if (trampoline == nullptr) { + trampoline = generator(); + slot.store(trampoline, std::memory_order_release); + } } + return trampoline; +} + +#if defined(_WIN32) && defined(CINDER_X86_64) +// Bridges the Microsoft x64 sret ABI to the JIT's internal reentry ABI. +// +// The JIT "reentry with processed args" entry point expects the plain +// vectorcall convention (RCX=callable, RDX=args, R8=nargsf, R9=kwnames) and +// returns its two result values in RAX:RDX (or XMM0:XMM1 for functions which +// return a primitive double). The C++ runtime helpers that re-dispatch through +// the reentry (e.g. rt::callWithIncorrectArgcount) are typed to return the +// 16-byte rt::StaticCallReturn / rt::StaticCallFPReturn structs. On the MS +// x64 ABI a struct larger than 8 bytes is returned via a hidden sret pointer in +// RCX, which shifts every argument by one register -- so without this bridge +// the reentry would read the callable (in RDX) as its args array and crash. +// +// This trampoline is itself invoked as a 16-byte-struct-returning function, so +// at entry the registers are: +// RCX = hidden sret buffer pointer +// RDX = reentry entry point +// R8 = callable (PyObject*) +// R9 = args (PyObject**) +// [RSP+0x28] = nargsf +// [RSP+0x30] = kwnames +// It rearranges these into the plain vectorcall ABI, calls the reentry, stores +// the two return values into the sret buffer, and returns the buffer in RAX (as +// required for an sret return on the MS x64 ABI). +void* generateStaticReentryTrampoline(bool fp) { CodeHolder code; - ICodeAllocator* code_allocator = mod_state->codeAllocator(); - code.init(code_allocator->asmJitEnvironment()); + ICodeAllocator* code_allocator = + cinderx::getModuleState()->code_allocator.get(); + const char* name = + fp ? "static_reentry_trampoline_fp" : "static_reentry_trampoline"; + ASM_CHECK(code.init(code_allocator->asmJitEnvironment()), name); arch::Builder a(&code); - Annotations annot; -#if defined(CINDER_X86_64) - auto annot_cursor = a.cursor(); - - a.push(x86::rbp); - a.mov(x86::rbp, x86::rsp); - - // save incoming arg registers - a.push(x86::r9); - a.push(x86::r8); - a.push(x86::rcx); - a.push(x86::rdx); - a.push(x86::rsi); - a.push(x86::rdi); - - annot.add("saveRegisters", &a, annot_cursor); - - // r10 contains the function object from our stub - a.mov(x86::rdi, x86::r10); - a.mov(x86::rsi, x86::rsp); - a.call(reinterpret_cast(JITRT_FailedDeferredCompileShim)); - a.leave(); + a.push(x86::rbx); // preserve callee-saved RBX; holds the sret buffer + a.mov(x86::rbx, x86::rcx); // RBX = sret buffer (survives the call) + a.mov(x86::rax, x86::rdx); // RAX = reentry target (scratch) + a.mov(x86::rcx, x86::r8); // RCX = callable (vectorcall arg 0) + a.mov(x86::rdx, x86::r9); // RDX = args (vectorcall arg 1) + // The two stack args are 8 bytes higher than at entry due to the pushed RBX. + a.mov(x86::r8, x86::ptr(x86::rsp, 0x30)); // R8 = nargsf (vectorcall arg 2) + a.mov(x86::r9, x86::ptr(x86::rsp, 0x38)); // R9 = kwnames (vectorcall arg 3) + a.sub(x86::rsp, 0x20); // shadow space (keeps RSP 16-byte aligned at the call) + a.call(x86::rax); + a.add(x86::rsp, 0x20); + if (fp) { + a.movsd(x86::ptr(x86::rbx, 0), x86::xmm0); + a.movsd(x86::ptr(x86::rbx, 8), x86::xmm1); + } else { + a.mov(x86::ptr(x86::rbx, 0), x86::rax); + a.mov(x86::ptr(x86::rbx, 8), x86::rdx); + } + a.mov(x86::rax, x86::rbx); // return the sret buffer pointer in RAX + a.pop(x86::rbx); a.ret(); -#elif defined(CINDER_AARCH64) - auto annot_cursor = a.cursor(); - - a.stp(arch::fp, arch::lr, a64::ptr_pre(a64::sp, -16)); - a.mov(arch::fp, a64::sp); - - // save incoming arg registers - a.stp(a64::x0, a64::x1, a64::ptr_pre(a64::sp, -64)); - a.stp(a64::x2, a64::x3, a64::ptr(a64::sp, 16)); - a.stp(a64::x4, a64::x5, a64::ptr(a64::sp, 32)); - a.stp(a64::x6, a64::x7, a64::ptr(a64::sp, 48)); - - annot.add("saveRegisters", &a, annot_cursor); - - // x10 contains the function object from our stub - a.mov(a64::x0, a64::x10); - a.mov(a64::x1, a64::sp); - a.mov(arch::reg_scratch_br, JITRT_FailedDeferredCompileShim); - a.blr(arch::reg_scratch_br); - a.mov(a64::sp, arch::fp); - a.ldp(arch::fp, arch::lr, a64::ptr_post(a64::sp, 16)); - a.ret(arch::lr); -#else - CINDER_UNSUPPORTED -#endif - - const char* name = "failedDeferredCompileTrampoline"; - void* result = finalizeCode(a, name); - - JIT_LOGIF( - getConfig().log.dump_asm, - "Disassembly for {}\n{}", - name, - annot.disassemble(result, code)); - - auto code_size = code.textSection()->realSize(); - register_raw_debug_symbol(name, __FILE__, __LINE__, result, code_size, 0); - std::vector> code_sections; - forEachSection([&](CodeSection section) { - auto asmjit_section = code.sectionByName(codeSectionName(section)); - if (asmjit_section == nullptr || asmjit_section->realSize() == 0) { - return; - } - auto section_start = static_cast(result) + asmjit_section->offset(); - code_sections.emplace_back( - reinterpret_cast(section_start), asmjit_section->realSize()); - }); - perf::registerFunction(code_sections, name); - return result; + return finalizeCode(a, name); } +#endif class AsmJitException : public std::exception { public: @@ -982,124 +661,31 @@ class ThrowableErrorHandler : public ErrorHandler { } }; -#if defined(CINDER_AARCH64) -// Save a set of callee-saved registers to the stack, properly handling both -// GP (x) and VecD (d) registers. GP and VecD registers must not be mixed in -// a single stp instruction. -void saveCalleeSavedRegsAarch64(arch::Builder* as, PhyRegisterSet saved_regs) { - auto gp_regs = saved_regs & ALL_GP_REGISTERS; - auto vecd_regs = saved_regs & ALL_VECD_REGISTERS; - - // Save GP registers first (they will be at higher addresses, restored last). - if (!gp_regs.Empty()) { - if (gp_regs.count() % 2 == 1) { - as->str(a64::x(gp_regs.GetFirst().loc), a64::ptr_pre(a64::sp, -16)); - gp_regs.RemoveFirst(); - } - while (!gp_regs.Empty()) { - auto first = a64::x(gp_regs.GetFirst().loc); - gp_regs.RemoveFirst(); - auto second = a64::x(gp_regs.GetFirst().loc); - gp_regs.RemoveFirst(); - as->stp(first, second, a64::ptr_pre(a64::sp, -16)); - } - } - - // Save VecD registers (they will be at lower addresses, restored first). - if (!vecd_regs.Empty()) { - if (vecd_regs.count() % 2 == 1) { - as->str( - a64::d(vecd_regs.GetFirst().loc - VECD_REG_BASE), - a64::ptr_pre(a64::sp, -16)); - vecd_regs.RemoveFirst(); - } - while (!vecd_regs.Empty()) { - auto first = a64::d(vecd_regs.GetFirst().loc - VECD_REG_BASE); - vecd_regs.RemoveFirst(); - auto second = a64::d(vecd_regs.GetFirst().loc - VECD_REG_BASE); - vecd_regs.RemoveFirst(); - as->stp(first, second, a64::ptr_pre(a64::sp, -16)); - } - } -} - -// Restore a set of callee-saved registers from the stack, in reverse order -// of saveCalleeSavedRegsAarch64. -void restoreCalleeSavedRegsAarch64( - arch::Builder* as, - PhyRegisterSet saved_regs) { - auto gp_regs = saved_regs & ALL_GP_REGISTERS; - auto vecd_regs = saved_regs & ALL_VECD_REGISTERS; - - // Restore VecD registers first (they were saved last, so they're at the - // lowest addresses). - if (!vecd_regs.Empty()) { - // Restore in reverse order (GetLast first). - // If odd count, the first-saved was a single str, so it's the last to - // restore and will be a single ldr. - bool odd = vecd_regs.count() % 2 == 1; - // First restore the pairs (from the paired stps). - // The pairs were saved GetFirst-first, so we restore GetLast-first. - PhyRegisterSet vecd_pairs = vecd_regs; - if (odd) { - vecd_pairs.RemoveFirst(); // skip the odd one for now - } - while (!vecd_pairs.Empty()) { - auto second = a64::d(vecd_pairs.GetLast().loc - VECD_REG_BASE); - vecd_pairs.RemoveLast(); - auto first = a64::d(vecd_pairs.GetLast().loc - VECD_REG_BASE); - vecd_pairs.RemoveLast(); - as->ldp(first, second, a64::ptr_post(a64::sp, 16)); - } - if (odd) { - as->ldr( - a64::d(vecd_regs.GetFirst().loc - VECD_REG_BASE), - a64::ptr_post(a64::sp, 16)); - } - } - - // Restore GP registers (they were saved first, so they're at higher - // addresses). - if (!gp_regs.Empty()) { - bool odd = gp_regs.count() % 2 == 1; - PhyRegisterSet gp_pairs = gp_regs; - if (odd) { - gp_pairs.RemoveFirst(); - } - while (!gp_pairs.Empty()) { - auto second = a64::x(gp_pairs.GetLast().loc); - gp_pairs.RemoveLast(); - auto first = a64::x(gp_pairs.GetLast().loc); - gp_pairs.RemoveLast(); - as->ldp(first, second, a64::ptr_post(a64::sp, 16)); - } - if (odd) { - as->ldr(a64::x(gp_regs.GetFirst().loc), a64::ptr_post(a64::sp, 16)); - } - } -} -#endif - } // namespace -NativeGenerator::NativeGenerator(const hir::Function* func) - : NativeGenerator{ - func, - generateDeoptTrampoline(false), - generateDeoptTrampoline(true), - generateFailedDeferredCompileTrampoline()} {} +void* getStaticReentryTrampoline(bool fp) { +#if defined(_WIN32) && defined(CINDER_X86_64) + // Lazily generate and cache once. Magic-static initialization is + // thread-safe, and the code allocator is always ready by the time + // JIT-compiled code (which is the only caller) runs. + if (fp) { + static void* trampoline = generateStaticReentryTrampoline(true); + return trampoline; + } + static void* trampoline = generateStaticReentryTrampoline(false); + return trampoline; +#else + (void)fp; + JIT_ABORT("static reentry trampoline is only needed on Windows x64"); +#endif +} NativeGenerator::NativeGenerator( const hir::Function* func, - void* deopt_trampoline, - void* deopt_trampoline_generators, - void* failed_deferred_compile_trampoline) + NativeGeneratorFactory& factory) : func_{func}, - deopt_trampoline_{deopt_trampoline}, - deopt_trampoline_generators_{deopt_trampoline_generators}, - failed_deferred_compile_trampoline_{failed_deferred_compile_trampoline}, - frame_asm_{func, env_}, - inline_stack_size_{calcInlineStackSize(func)} { + inline_stack_size_{calcInlineStackSize(func)}, + factory_(factory) { env_.has_inlined_functions = inline_stack_size_ > 0; } @@ -1115,7 +701,6 @@ PhyLocation get_arg_location_phy_location(int arg) { } JIT_ABORT("only six first registers should be used"); - return 0; } std::span NativeGenerator::getCodeBuffer() const { @@ -1132,7 +717,8 @@ void* NativeGenerator::getVectorcallEntry() { JIT_CHECK(as_ == nullptr, "Builder should not have been initialized."); CodeHolder code; - ICodeAllocator* code_allocator = cinderx::getModuleState()->codeAllocator(); + ICodeAllocator* code_allocator = + cinderx::getModuleState()->code_allocator.get(); code.init(code_allocator->asmJitEnvironment()); ThrowableErrorHandler eh; code.setErrorHandler(&eh); @@ -1148,22 +734,22 @@ void* NativeGenerator::getVectorcallEntry() { } as_ = new arch::Builder(&code); - frame_asm_.setAssembler(as_); env_.as = as_; env_.hard_exit_label = as_->newLabel(); env_.gen_resume_entry_label = as_->newLabel(); + env_.is_generator = isGen(); // Prepare the location for where our arguments will go. This just // uses general purpose registers while available for non-floating // point values, and floating point values while available for fp // arguments. - const std::vector& checks = GetFunction()->typed_args; + const std::vector& checks = getFunction()->typed_args; // gp_index starts at 1 because the first argument is reserved for the // function for (size_t i = 0, check_index = 0, gp_index = 1, fp_index = 0; - i < static_cast(GetFunction()->numArgs()); + i < static_cast(getFunction()->numArgs()); i++) { auto add_gp = [&]() { if (gp_index < ARGUMENT_REGS.size()) { @@ -1193,89 +779,131 @@ void* NativeGenerator::getVectorcallEntry() { add_gp(); } - auto func = GetFunction(); + auto func = getFunction(); env_.ctx = getContext(); + env_.reifier = func->env.reifier; env_.code_rt = env_.ctx->allocateCodeRuntime( func->code.get(), func->builtins.get(), func->globals.get()); -#if defined(ENABLE_LIGHTWEIGHT_FRAMES) && PY_VERSION_HEX >= 0x030E0000 - env_.code_rt->setReifier(func->reifier); -#endif + env_.addReference(func->code.getObj()); + env_.addReference(func->builtins.getObj()); + env_.addReference(func->globals.getObj()); for (auto& ref : func->env.references()) { - env_.code_rt->addReference(ref); + env_.addReference(ref); } - lir::LIRGenerator lirgen(GetFunction(), &env_); + lir::LIRGenerator lirgen(getFunction(), &env_); std::unique_ptr lir_func; +#if defined(CINDER_X86_64) && defined(_WIN32) + { + int fh_size = jit::frameHeaderSize(func_->code) + sizeof(void*); + env_.win_struct_ret_offset = -(fh_size + inline_stack_size_ + 16); + } +#endif + COMPILE_TIMER( - GetFunction()->compilation_phase_timer, + getFunction()->compilation_phase_timer, "Lowering into LIR", - lir_func = lirgen.TranslateFunction()) + lir_func = lirgen.translateFunction()) + checkLirSize(*lir_func); JIT_LOGIF( getConfig().log.dump_lir, "LIR for {} after generation:\n{}", - GetFunction()->fullname, + getFunction()->fullname, *lir_func); PostGenerationRewrite post_gen(lir_func.get(), &env_); COMPILE_TIMER( - GetFunction()->compilation_phase_timer, + getFunction()->compilation_phase_timer, "LIR transformations", post_gen.run()) JIT_LOGIF( getConfig().log.dump_lir, "LIR for {} after postgen rewrites:\n{}", - GetFunction()->fullname, + getFunction()->fullname, *lir_func); COMPILE_TIMER( - GetFunction()->compilation_phase_timer, + getFunction()->compilation_phase_timer, "DeadCodeElimination", eliminateDeadCode(lir_func.get())) - LinearScanAllocator lsalloc( - lir_func.get(), frame_asm_.frameHeaderSize() + inline_stack_size_); + COMPILE_TIMER( + getFunction()->compilation_phase_timer, + "Target Selection and Legalization", + selectTargetOpcodes(lir_func.get())) + + JIT_LOGIF( + getConfig().log.dump_lir, + "LIR for {} after target selection and legalization:\n{}", + getFunction()->fullname, + *lir_func); + + int frame_header_size = frameHeaderSize(func_->code); + frame_header_size += sizeof(void*); + + int reserved_stack_space = frame_header_size + inline_stack_size_; +#if defined(CINDER_X86_64) && defined(_WIN32) + reserved_stack_space += 16; +#endif + + std::unique_ptr allocator; + switch (getConfig().reg_alloc) { + case RegAllocKind::kLinearScan: + allocator = std::make_unique( + lir_func.get(), reserved_stack_space); + break; + case RegAllocKind::kSpill: + allocator = std::make_unique( + lir_func.get(), reserved_stack_space); + break; + } COMPILE_TIMER( - GetFunction()->compilation_phase_timer, + getFunction()->compilation_phase_timer, "Register Allocation", - lsalloc.run()) + allocator->run()) - env_.shadow_frames_and_spill_size = lsalloc.getFrameSize(); - env_.changed_regs = lsalloc.getChangedRegs(); + env_.shadow_frames_and_spill_size = allocator->getFrameSize(); + env_.changed_regs = allocator->getChangedRegs(); env_.exit_label = as_->newLabel(); - env_.exit_for_yield_label = as_->newLabel(); - env_.frame_mode = GetFunction()->frameMode; - if (GetFunction()->code->co_flags & kCoFlagsAnyGenerator) { - env_.initial_yield_spill_size_ = lsalloc.initialYieldSpillSize(); - } + env_.can_deopt = getFunction()->canDeopt(); JIT_LOGIF( getConfig().log.dump_lir, "LIR for {} after register allocation:\n{}", - GetFunction()->fullname, + getFunction()->fullname, *lir_func); PostRegAllocRewrite post_rewrite(lir_func.get(), &env_); COMPILE_TIMER( - GetFunction()->compilation_phase_timer, + getFunction()->compilation_phase_timer, "Post Reg Alloc Rewrite", post_rewrite.run()) +#if defined(CINDER_AARCH64) + // Peepholes go last, once nothing else will add to or remove from the + // instruction stream. + COMPILE_TIMER( + getFunction()->compilation_phase_timer, + "Post Reg Alloc Peephole", + runPostRegAllocPeephole(lir_func.get())) +#endif + JIT_LOGIF( getConfig().log.dump_lir, "LIR for {} after postalloc rewrites:\n{}", - GetFunction()->fullname, + getFunction()->fullname, *lir_func); if (!verifyPostRegAllocInvariants(lir_func.get(), std::cerr)) { JIT_ABORT( "LIR for {} failed verification:\n{}", - GetFunction()->fullname, + getFunction()->fullname, *lir_func); } @@ -1283,17 +911,18 @@ void* NativeGenerator::getVectorcallEntry() { try { COMPILE_TIMER( - GetFunction()->compilation_phase_timer, + getFunction()->compilation_phase_timer, "Code Generation", - generateCode(code)) + generateCode(code, lirgen.frameSetupBlock())) } catch (const AsmJitException& ex) { String s; FormatOptions formatOptions; + formatOptions.setFlags(FormatFlags::kHexImms); Formatter::formatNodeList(s, formatOptions, as_); JIT_ABORT( "Failed to emit code for '{}': '{}' failed with '{}'\n\n" "Builder contents on failure:\n{}", - GetFunction()->fullname, + getFunction()->fullname, ex.expr, ex.message, s.data()); @@ -1307,6 +936,13 @@ void* NativeGenerator::getVectorcallEntry() { JIT_DCHECK(code.codeSize() < INT_MAX, "Code size is larger than INT_MAX"); compiled_size_ = code.codeSize(); env_.code_rt->setFrameSize(env_.stack_frame_size); + if (getFunction()->code->co_flags & kCoFlagsAnyGenerator) { + JIT_DCHECK( + env_.shadow_frames_and_spill_size % kPointerSize == 0, + "Bad spill alignment"); + env_.code_rt->setSpillWords( + env_.shadow_frames_and_spill_size / kPointerSize); + } return vectorcall_entry_; } @@ -1321,11 +957,11 @@ void* NativeGenerator::getStaticEntry() { JITRT_STATIC_ENTRY_OFFSET); } -int NativeGenerator::GetCompiledFunctionStackSize() const { +int NativeGenerator::getCompiledFunctionStackSize() const { return env_.stack_frame_size; } -int NativeGenerator::GetCompiledFunctionSpillStackSize() const { +int NativeGenerator::getCompiledFunctionSpillStackSize() const { return spill_stack_size_; } @@ -1334,7 +970,7 @@ void NativeGenerator::generateFunctionEntry() { as_->push(x86::rbp); as_->mov(x86::rbp, x86::rsp); #elif defined(CINDER_AARCH64) - as_->stp(arch::fp, arch::lr, a64::ptr_pre(a64::sp, -16)); + as_->stp(arch::fp, arch::lr, a64::ptr_pre(a64::sp, -arch::kFrameRecordSize)); as_->mov(arch::fp, a64::sp); #else CINDER_UNSUPPORTED @@ -1347,7 +983,7 @@ void NativeGenerator::generateFunctionExit() { as_->ret(); #elif defined(CINDER_AARCH64) as_->mov(a64::sp, arch::fp); - as_->ldp(arch::fp, arch::lr, a64::ptr_post(a64::sp, 16)); + as_->ldp(arch::fp, arch::lr, a64::ptr_post(a64::sp, arch::kFrameRecordSize)); as_->ret(arch::lr); #else CINDER_UNSUPPORTED @@ -1385,7 +1021,7 @@ NativeGenerator::FrameInfo NativeGenerator::computeFrameInfo() { .header_and_spill_size = std::max(env_.shadow_frames_and_spill_size, kPointerSize), .saved_regs = env_.changed_regs & CALLEE_SAVE_REGS, - .arg_buffer_size = env_.max_arg_buffer_size, + .arg_buffer_size = env_.max_arg_buffer_size + env_.reserve_stack_size, }; if ((info.header_and_spill_size + info.saved_regs_size() + info.arg_buffer_size) % @@ -1399,85 +1035,17 @@ NativeGenerator::FrameInfo NativeGenerator::computeFrameInfo() { return info; } -int NativeGenerator::allocateHeaderAndSpillSpace(const FrameInfo& frame_info) { -#if defined(CINDER_X86_64) - int padding = frame_info.header_and_spill_size % kStackAlign; - as_->sub(x86::rsp, frame_info.header_and_spill_size + padding); - return padding; -#elif defined(CINDER_AARCH64) - int modulo = frame_info.header_and_spill_size % kStackAlign; - int padding = modulo == 0 ? 0 : kStackAlign - modulo; - as_->sub(a64::sp, a64::sp, frame_info.header_and_spill_size + padding); - - // There is a difference here from x86-64, because the aarch64 stack cannot be - // misaligned. Here we are returning the amount of space that we have added to - // keep the stack aligned, as opposed to the amount of space that we have gone - // over the stack alignment. - return padding; -#else - CINDER_UNSUPPORTED - return 0; -#endif -} - -void NativeGenerator::saveCallerRegisters( - const FrameInfo& frame_info, - [[maybe_unused]] arch::Gp tstate_reg) { -#if defined(CINDER_X86_64) -#ifdef ENABLE_SHADOW_FRAMES - frame_asm_.initializeFrameHeader(tstate_reg, x86::rax); -#endif - // Push used callee-saved registers. - auto saved_regs = frame_info.saved_regs; - while (!saved_regs.Empty()) { - as_->push(x86::gpq(saved_regs.GetFirst().loc)); - saved_regs.RemoveFirst(); - } - - if (frame_info.arg_buffer_size > 0) { - as_->sub(x86::rsp, frame_info.arg_buffer_size); - } -#elif defined(CINDER_AARCH64) -#ifdef ENABLE_SHADOW_FRAMES - frame_asm_.initializeFrameHeader(tstate_reg, a64::x0); -#endif - // Push used callee-saved registers. - saveCalleeSavedRegsAarch64(as_, frame_info.saved_regs); - - if (frame_info.arg_buffer_size > 0) { - JIT_CHECK(frame_info.arg_buffer_size % kStackAlign == 0, "unaligned"); - as_->sub(a64::sp, a64::sp, frame_info.arg_buffer_size); - } -#else - CINDER_UNSUPPORTED -#endif -} - -void NativeGenerator::setupFrameAndSaveCallerRegisters( - const FrameInfo& frame_info, - arch::Gp tstate_reg) { -#if defined(CINDER_X86_64) - as_->sub(x86::rsp, frame_info.header_and_spill_size); -#elif defined(CINDER_AARCH64) - JIT_CHECK(frame_info.header_and_spill_size % kStackAlign == 0, "unaligned"); - as_->sub(a64::sp, a64::sp, frame_info.header_and_spill_size); -#else - CINDER_UNSUPPORTED -#endif - saveCallerRegisters(frame_info, tstate_reg); -} - arch::Gp get_arg_location(int arg) { #if defined(CINDER_X86_64) auto phyloc = get_arg_location_phy_location(arg); - if (phyloc.is_register()) { + if (phyloc.isRegister()) { return x86::gpq(phyloc.loc); } #elif defined(CINDER_AARCH64) auto phyloc = get_arg_location_phy_location(arg); - if (phyloc.is_register()) { + if (phyloc.isRegister()) { return a64::x(phyloc.loc); } #else @@ -1487,867 +1055,6 @@ arch::Gp get_arg_location(int arg) { JIT_ABORT("should only be used with first six args"); } -bool NativeGenerator::linkFrameNeedsSpill() { - if (!isGen()) { - return true; - } - - // On 3.12 we link the frame immediately so we need to preserve the - // arguments for generators as well. - if constexpr (PY_VERSION_HEX < 0x030C0000) { - return false; - } - return true; -} - -void NativeGenerator::generatePrologue( - const FrameInfo& frame_info, - Label correct_arg_count, - Label finish_frame_setup) { -#if defined(CINDER_X86_64) - // The boxed return wrapper gets generated first, if it is necessary. - auto [generic_entry_cursor, box_entry_cursor] = generateBoxedReturnWrapper(); - - generateFunctionEntry(); - - // Verify arguments have been passed in correctly. - if (func_->has_primitive_args) { - generatePrimitiveArgsPrologue(); - } else { - generateArgcountCheckPrologue(correct_arg_count); - } - as_->bind(correct_arg_count); - - Label setup_frame = as_->newLabel(); - - if (hasStaticEntry()) { - if (!func_->has_primitive_args) { - // We weren't called statically, but we've now resolved all arguments to - // fixed offsets. Validate that the arguments are correctly typed. - generateStaticMethodTypeChecks(setup_frame); - } else if (func_->has_primitive_first_arg) { - as_->mov(x86::rdx, 0); - } - } - - env_.addAnnotation("Generic entry", generic_entry_cursor); - - if (box_entry_cursor) { - env_.addAnnotation( - "Generic entry (box primitive return)", box_entry_cursor); - } - - // Args are now validated, setup frame. - constexpr auto kFuncPtrReg = x86::gpq(INITIAL_FUNC_REG.loc); - constexpr auto kArgsReg = x86::gpq(INITIAL_EXTRA_ARGS_REG.loc); - constexpr auto kArgsPastSixReg = kArgsReg; - - asmjit::BaseNode* frame_cursor = as_->cursor(); - as_->bind(setup_frame); - std::vector> save_regs; - - save_regs.emplace_back(x86::rsi, kArgsReg); - if (GetFunction()->uses_runtime_func) { - save_regs.emplace_back(x86::rdi, kFuncPtrReg); - } - - // Ensure that rsp is below the fields in the stack allocated interpreter - // frame that may be initialized in the `generateLinkFrame` call below, - // preventing the signal handling routine in the kernel from overwriting - // them. - int padding = allocateHeaderAndSpillSpace(frame_info); - - frame_asm_.generateLinkFrame( - kFuncPtrReg, x86::gpq(INITIAL_TSTATE_REG.loc), save_regs); - - env_.addAnnotation("Link frame", frame_cursor); - - asmjit::BaseNode* load_args_cursor = as_->cursor(); - // Move arguments into their expected registers and then set a register as the - // base for additional args. - bool has_extra_args = false; - for (size_t i = 0; i < env_.arg_locations.size(); i++) { - PhyLocation arg = env_.arg_locations[i]; - if (arg == PhyLocation::REG_INVALID) { - has_extra_args = true; - continue; - } - if (arg.is_gp_register()) { - as_->mov(x86::gpq(arg.loc), x86::ptr(kArgsReg, i * sizeof(void*))); - } else { - as_->movsd(x86::xmm(arg.loc), x86::ptr(kArgsReg, i * sizeof(void*))); - } - } - if (has_extra_args) { - // Load the location of the remaining args, the backend will deal with - // loading them from here... - as_->lea( - kArgsPastSixReg, - x86::ptr(kArgsReg, (ARGUMENT_REGS.size() - 1) * sizeof(void*))); - } - env_.addAnnotation("Load arguments", load_args_cursor); - - // We already allocated stack space for the header and spill data, clean - // up any alignment padding we added - if (padding) { - as_->add(x86::rsp, padding); - } - - // Finally allocate the saved space required for the actual function. - auto finish_frame_setup_cursor = as_->cursor(); - as_->bind(finish_frame_setup); - saveCallerRegisters(frame_info, x86::r11); - - env_.addAnnotation("Finish frame setup", finish_frame_setup_cursor); -#elif defined(CINDER_AARCH64) - // The boxed return wrapper gets generated first, if it is necessary. - auto [generic_entry_cursor, box_entry_cursor] = generateBoxedReturnWrapper(); - - generateFunctionEntry(); - - // Verify arguments have been passed in correctly. - if (func_->has_primitive_args) { - generatePrimitiveArgsPrologue(); - } else { - generateArgcountCheckPrologue(correct_arg_count); - } - as_->bind(correct_arg_count); - - Label setup_frame = as_->newLabel(); - - if (hasStaticEntry()) { - if (!func_->has_primitive_args) { - // We weren't called statically, but we've now resolved all arguments to - // fixed offsets. Validate that the arguments are correctly typed. - generateStaticMethodTypeChecks(setup_frame); - } else if (func_->has_primitive_first_arg) { - as_->mov(a64::x2, 0); - } - } - - env_.addAnnotation("Generic entry", generic_entry_cursor); - - if (box_entry_cursor) { - env_.addAnnotation( - "Generic entry (box primitive return)", box_entry_cursor); - } - - // Args are now validated, setup frame. - constexpr auto kFuncPtrReg = a64::x(INITIAL_FUNC_REG.loc); - constexpr auto kArgsReg = a64::x(INITIAL_EXTRA_ARGS_REG.loc); - constexpr auto kArgsPastEightReg = kArgsReg; - - asmjit::BaseNode* frame_cursor = as_->cursor(); - as_->bind(setup_frame); - std::vector> save_regs; - - save_regs.emplace_back(a64::x1, kArgsReg); - if (GetFunction()->uses_runtime_func) { - save_regs.emplace_back(a64::x0, kFuncPtrReg); - } - - // Ensure that sp is below the fields in the stack allocated interpreter - // frame that may be initialized in the `generateLinkFrame` call below, - // preventing the signal handling routine in the kernel from overwriting - // them. Note that we do not need to worry about the padding here, as the - // stack is already aligned by that allocation function. - (void)allocateHeaderAndSpillSpace(frame_info); - - frame_asm_.generateLinkFrame( - kFuncPtrReg, a64::x(INITIAL_TSTATE_REG.loc), save_regs); - - env_.addAnnotation("Link frame", frame_cursor); - - asmjit::BaseNode* load_args_cursor = as_->cursor(); - // Move arguments into their expected registers and then set a register as the - // base for additional args. - bool has_extra_args = false; - for (size_t i = 0; i < env_.arg_locations.size(); i++) { - PhyLocation arg = env_.arg_locations[i]; - if (arg == PhyLocation::REG_INVALID) { - has_extra_args = true; - continue; - } - if (arg.is_gp_register()) { - as_->ldr( - a64::x(arg.loc), - arch::ptr_resolve( - as_, kArgsReg, i * sizeof(void*), arch::reg_scratch_0)); - } else { - as_->ldr( - a64::d(arg.loc), - arch::ptr_resolve( - as_, kArgsReg, i * sizeof(void*), arch::reg_scratch_0)); - } - } - if (has_extra_args) { - // Load the location of the remaining args, the backend will deal with - // loading them from here... - as_->add( - kArgsPastEightReg, - kArgsReg, - (ARGUMENT_REGS.size() - 1) * sizeof(void*)); - } - env_.addAnnotation("Load arguments", load_args_cursor); - - // Finally allocate the saved space required for the actual function. - auto finish_frame_setup_cursor = as_->cursor(); - as_->bind(finish_frame_setup); - saveCallerRegisters(frame_info, a64::x11); - - env_.addAnnotation("Finish frame setup", finish_frame_setup_cursor); -#else - CINDER_UNSUPPORTED -#endif -} - -static void -emitCompare(arch::Builder* as, arch::Gp lhs, void* rhs, arch::Gp scratch) { -#if defined(CINDER_X86_64) - uint64_t rhsi = reinterpret_cast(rhs); - - if (!fitsSignedInt<32>(rhsi)) { - // in shared mode type can be in a high address - as->mov(scratch, rhsi); - as->cmp(lhs, scratch); - } else { - as->cmp(lhs, rhsi); - } -#elif defined(CINDER_AARCH64) - uint64_t rhsi = reinterpret_cast(rhs); - - if (!a64::Utils::isAddSubImm(rhsi)) { - as->mov(scratch, rhsi); - as->cmp(lhs, scratch); - } else { - as->cmp(lhs, rhsi); - } -#else - CINDER_UNSUPPORTED -#endif -} - -void NativeGenerator::generateStaticMethodTypeChecks(Label setup_frame) { - // JITRT_CallWithIncorrectArgcount uses the fact that our checks are set up - // from last to first argument - we order the jumps so that the common case of - // no defaulted arguments comes first, and end up with the following - // structure: generic entry: compare defaulted arg count to 0 if zero: go to - // first check compare defaulted arg count to 1 if zero: go to second check - // ... - // This is complicated a bit by the fact that not every argument will have a - // check, as we elide the dynamic ones. For that, we do bookkeeping and assign - // all defaulted arg counts up to the next local to the same label. - const std::vector& checks = GetFunction()->typed_args; - env_.static_arg_typecheck_failed_label = as_->newLabel(); - if (!checks.size()) { - return; - } - -#if defined(CINDER_X86_64) - // We build a vector of labels corresponding to [first_check, second_check, - // ..., setup_frame] which will have |checks| + 1 elements, and the - // first_check label will precede the first check. - auto table_label = as_->newLabel(); - as_->lea(x86::r8, x86::ptr(table_label)); - as_->lea(x86::r8, x86::ptr(x86::r8, x86::rcx, 3)); - as_->jmp(x86::r8); - auto jump_table_cursor = as_->cursor(); - as_->align(AlignMode::kCode, 8); - as_->bind(table_label); - std::vector