cpython.BUILD.bazel

load("@//bazel/rules:foreign_cc_shared_wrapper.bzl", "foreign_cc_shared_wrapper")
load("@@//bazel/rules/dd_packaging:dd_cc_packaged.bzl", "dd_cc_packaged")
load("@@//bazel/rules/dd_packaging:dd_collect_dependencies.bzl", "dd_collect_dependencies")
load("@@//bazel/rules/foreign_cc_runnable:foreign_cc_runnable.bzl", "foreign_cc_runnable")
load("@//deps/openssl:version.bzl", "OPENSSL_VERSION")
load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory")
load("@bazel_lib//lib:run_binary.bzl", "run_binary")
load("@bazel_skylib//rules:select_file.bzl", "select_file")
load("@cpython_versions//:constants.bzl", "PYTHON_MAJOR_MINOR", "PYTHON_MAJOR_MINOR_FLAT")
load("@rules_cc//cc:defs.bzl", "cc_import", "cc_library")
load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make")
load("@rules_pkg//pkg:install.bzl", "pkg_install")
load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files", "pkg_mklink")

# Keep in sync with the ones on repos.MODULE.bazel
python_externals = {
    "bzip2": "1.0.8",
    "mpdecimal": "4.0.0",
    "sqlite": "3.53.2.0",
    "xz": "5.2.5",
    "zlib": "1.3.1",
    "libffi": "3.4.4",
    "tcltk": "8.6.15.0",
}

# These rules will make it easier to get a reference to their folder via $(location)
# and they add the version in the folder name (as some of the vcxproj stuff relies on that)
[
    copy_to_directory(
        name = "{}_win_dir".format(dep),
        srcs = ["@{}_win".format(dep)],
        out = "{}-{}".format(dep, version),
        include_external_repositories = ["*"],
    )
    for dep, version in python_externals.items()
]

# This sets up OpenSSL files in a layout such as the Python build system for Windows expects
copy_to_directory(
    name = "openssl-bin_win_dir",
    srcs = ["@openssl"],
    # TODO(team:agent-build): Single source of truth for dependency versions
    out = "openssl-bin-{}/amd64".format(OPENSSL_VERSION),
    include_external_repositories = ["*openssl*"],
    # This reproduces the expected layout (libs at root, includes under `include`)
    root_paths = [
        "openssl/bin",
        "openssl/lib",
        "openssl",
    ],
)

run_binary(
    name = "python_win",
    srcs = (
        glob(
            ["**"],
            exclude = [
                "**/*.pyc",
                "BUILD.bazel",
                # Unix-only build helper injected into the CPython repository.
                # Keep it out of the Windows action key; the Windows build does not use it.
                "redacted_compat.h",
            ],
        ) + [
            "bzip2_win_dir",
            "mpdecimal_win_dir",
            "sqlite_win_dir",
            "xz_win_dir",
            "zlib_win_dir",
            "libffi_win_dir",
            "openssl-bin_win_dir",
            "tcltk_win_dir",
            "@python_3_12//:python3",
            # Hermetic compiler, SDK and MSBuild; nothing is taken from a host
            # Visual Studio installation.
            "@msvc_toolchains//msvc/tree:toolchain_tree",
            "@msvc_toolchains//msvc/tree:version_marker",
            "@msvc_toolchains//winsdk/tree:toolchain_tree",
            "@msvc_toolchains//winsdk/tree:root_marker",
            "@msvc_toolchains//msbuild:msbuild",
            "@msvc_toolchains//msbuild:toolchain_tree",
            "@msvc_toolchains//msbuild:vctargets_marker",
        ]
    ),
    env = {
        "PROCESSOR_ARCHITECTURE": "AMD64",
        # Input and output paths
        "SRCFILE": r"$(location LICENSE)",  # Any file at the root of the repo works here
        "OUTDIR": "$(@D)/build",
        # Paths to dependencies:
        "BZ2_DIR": "$(location bzip2_win_dir)",
        "MPDECIMAL_DIR": "$(location mpdecimal_win_dir)",
        "SQLITE3_DIR": "$(location sqlite_win_dir)",
        "XZ_DIR": "$(location xz_win_dir)",
        "ZLIB_DIR": "$(location zlib_win_dir)",
        "LIBFFI_DIR": "$(location libffi_win_dir)",
        "OPENSSL_DIR": r"$(location openssl-bin_win_dir)",
        "TCLTK_DIR": r"$(location tcltk_win_dir)\amd64",
        "TCL_VERSION": python_externals["tcltk"],
        "MSBUILD": "$(location @msvc_toolchains//msbuild:msbuild)",
        "PYTHON_FOR_BUILD": "$(location @python_3_12//:python3)",
        # Marker files; build_python.bat derives each toolchain root from them.
        "VCTOOLS_VERSION_FILE": "$(location @msvc_toolchains//msvc/tree:version_marker)",
        "WINSDK_MARKER": "$(location @msvc_toolchains//winsdk/tree:root_marker)",
        "VCTARGETS_MARKER": "$(location @msvc_toolchains//msbuild:vctargets_marker)",
    },
    outs = [
        "build/{}".format(file)
        for file in [
            "python.exe",
            "python3.dll",
            "python{}.dll".format(PYTHON_MAJOR_MINOR_FLAT),
            "pythonw.exe",
            "vcruntime140.dll",
            "vcruntime140_1.dll",
            "LICENSE.txt",
        ]
    ],
    out_dirs = [
        "build/{}".format(dir)
        for dir in [
            "DLLs",
            "include",
            "Lib",
            "libs",
            "Scripts",
        ]
    ],
    tool = "build_python.bat",
    target_compatible_with = ["@platforms//os:windows"],
    visibility = ["//visibility:public"],
)

select_file(
    name = "headers_win",
    srcs = "python_win",
    subpath = "build/include",
)

select_file(
    name = "python_win_lib",
    srcs = "python_win",
    subpath = "build/python{}.dll".format(PYTHON_MAJOR_MINOR_FLAT),
    visibility = ["//visibility:public"],
)

cc_import(
    name = "_python_dll",
    shared_library = "build/python{}.dll".format(PYTHON_MAJOR_MINOR_FLAT),
    visibility = ["//visibility:private"],
)

# The extra cc_library layer on top of the cc_import is needed so that
# relative paths (like the one for the includes) are resolved correctly
cc_library(
    name = "lib_win",
    hdrs = [":headers_win"],
    includes = ["build/include"],
    deps = [":_python_dll"],
    visibility = ["//visibility:public"],
)

filegroup(
    name = "all_srcs",
    srcs = glob(
        ["**"],
        exclude = [
            "BUILD.bazel",
            ".devcontainer/**",
            "Android/**",
            "Doc/**",
            "InternalDocs/**",
            "iOS/**",
            # Windows related.
            "PC/**",
            "PCbuild/**",
            "Tools/msi/**",
            "Tools/nuget/**",
            # Tests that don't need to be shipped with the Agent
            "Lib/idlelib/idle_test/**",
            "Lib/test/**",
        ],
    ),
)

UNIX_BINS = [
    "python{}".format(PYTHON_MAJOR_MINOR),
]

LINUX_SO = "libpython{}.so.1.0".format(PYTHON_MAJOR_MINOR)
MACOS_DYLIB = "libpython{}.dylib".format(PYTHON_MAJOR_MINOR)

python_dep_libs = {
    "libffi": "-lffi",
    "libsqlite3": "-lsqlite3",
    "zlib": "-lz",
    "bzip2": "-lbz2",
    "liblzma": "-llzma",
}

# On macOS, the hermetic SDK's /usr/lib is injected before Bazel dependency
# library search paths. Use exact files from rules_foreign_cc's staged deps
# directory so CPython extension modules cannot resolve to SDK libraries.
python_dep_libs_explicit = {
    "libffi": "$$EXT_BUILD_DEPS/lib/libffi.a",
    "libsqlite3": "$$EXT_BUILD_DEPS/lib/libsqlite3.dylib",
    "zlib": "$$EXT_BUILD_DEPS/lib/libz.dylib",
    "bzip2": "$$EXT_BUILD_DEPS/lib/libbz2.dylib",
    "liblzma": "$$EXT_BUILD_DEPS/lib/liblzma.dylib",
}

configure_make(
    name = "python_unix",
    configure_options = [
        "--enable-ipv6",
        "--with-ensurepip=no",  # pip is installed as a separate target
        "--enable-shared",
        "--without-static-libpython",
        "--with-dbmliborder=",
        "--with-openssl=$$EXT_BUILD_DEPS/openssl",
        "--with-openssl-rpath=yes",
        "--disable-test-modules",
    ] + select({
        "@@//:macos_arm64": ["--with-universal-archs=universal2"],
        "@@//:macos_x86_64": ["--with-universal-archs=intel"],
        "//conditions:default": [],
    }) + select({
        "@platforms//os:macos": [
            # Override ARFLAGS: the hermetic toolchain injects -static (invalid for llvm-ar).
            # Passing as a configure arg takes precedence over the inline env.
            "ARFLAGS=rcs",
        ],
        "//conditions:default": [],
    }),
    copts = [
        "-O2",
    ],
    env = {
        "OPT": "-DNDEBUG -fwrapv",
        # Ensure we don't use the system provided .pc
        "PKG_CONFIG_LIBDIR": "/does/not/exist",
        # Avoid quotation issues around keeping `redacted` (for __DATE__/__TIME__) properly quoted
        # (See comments on redacted_compat.h)
        # It's important to pass this on the `env` and not on `configure_options` to avoid CPPFLAGS
        # populated by rules_foreign_cc from being overwritten
        "CPPFLAGS": "-include$(execpath :redacted_compat.h)",
        # We need to pass linker flags here instead of `linkopts` because we don't want these to be
        # propagated to dependent targets, as `linkopts` would do.
        # This is meant to allow python to find its dependency during its modules import test.
        # We will use the install_dir rpath later on
        "LDFLAGS": "-Wl,-rpath,$$EXT_BUILD_DEPS/lib",
    } | {
        dep.upper() + "_CFLAGS": "-I$$EXT_BUILD_DEPS/include"
        for dep in python_dep_libs.keys()
    } | select({
        "@platforms//os:macos": {
            dep.upper() + "_LIBS": lib
            for dep, lib in python_dep_libs_explicit.items()
        },
        "//conditions:default": {
            dep.upper() + "_LIBS": lib
            for dep, lib in python_dep_libs.items()
        },
    }) | select({
        "@platforms//os:macos": {
            # `ar` defaults to `libtool` on darwin toolchains, which doesn't support the same flags
            # Setting `AR` allows us to use llvm-ar (coming from the hermetic toolchain) instead
            # See also: https://github.com/bazelbuild/bazel/issues/5127
            "AR": "$(execpath @llvm_toolchain_llvm//:ar)",
        },
        "//conditions:default": {},
    }),
    build_data = [
        ":redacted_compat.h",
        "@@//deps/cpython:fix_sysconfigdata.py",
        "@python_3_12//:python3",
    ] + select({
        "@platforms//os:macos": [
            "@llvm_toolchain_llvm//:ar",
        ],
        "//conditions:default": [],
    }),
    lib_source = ":all_srcs",
    out_binaries = UNIX_BINS,
    out_data_dirs = [
        "lib/python{}".format(PYTHON_MAJOR_MINOR),
    ],
    out_data_files = select({
        "@platforms//os:linux": [
            # Even though this is a shared lib, we generally don't want to link against this.
            # When linking against Python, one must decide between the "Stable" API (libpython3.so)
            # or the minor-version specific libpython3.y.so. In the context of the Agent, we've always
            # linked against the latter, but shipped both (in case there are extension modules requiring
            # the Stable API). Therefore, we mark this one as "data".
            # See https://peps.python.org/pep-0384/#linkage
            "lib/libpython3.so",
        ],
        "@platforms//os:macos": [],
    }),
    out_shared_libs = select({
        "@platforms//os:linux": [
            LINUX_SO,
        ],
        "@platforms//os:macos": [
            MACOS_DYLIB,
        ],
    }),
    out_include_dir = "include",
    visibility = ["//visibility:public"],
    deps = [
        "@bzip2//:libbz2",
        "@libffi//:ffi",
        "@openssl//:openssl",
        "@sqlite3//:libsqlite3",
        "@xz//:liblzma",
        "@zlib//:zlib",
    ],
    dynamic_deps = [
        "@bzip2//:bz2_pkg",
        "@openssl//:libssl_shared_pkg",
        "@openssl//:libcrypto_shared_pkg",
        "@sqlite3//:sqlite3_pkg",
        "@xz//:lzma_pkg",
        "@zlib//:z_pkg",
    ],
    includes = ["python{}".format(PYTHON_MAJOR_MINOR)],
    targets = [
        # Build in parallel but install without parallel execution
        # (see https://github.com/python/cpython/issues/109796)
        "-j 16",
        "install",
    ],
    # $(INSTALL_DIR) is the make-variable provided by //:install_dir.
    toolchains = ["@@//:install_dir"],
    target_compatible_with = select({
        "@platforms//os:macos": [],
        "@platforms//os:linux": [],
        "//conditions:default": ["@platforms//:incompatible"],
    }),
    # python's build system will output the entire build config to _sysconfigdata_xxx.py
    # This is later used to build extensions with the same tools/compiler/flags/config as the interpreter
    # However, in our case, that means using tools that are stored in the build sandbox, so they aren't
    # usable when building an extension, causing all builds to fail.
    # fix_sysconfigdata.py strips sandbox-absolute paths from known tool entries (leaving just the
    # basename so they can be found on PATH), clears build flags that are Bazel-specific, and replaces
    # sandbox paths with their install-prefix-relative equivalents.
    postfix_script = """
    $(execpath @python_3_12//:python3) $(execpath @@//deps/cpython:fix_sysconfigdata.py) \
      --install-prefix "$(INSTALL_DIR)/embedded" \
      --sandbox-prefix "$$BUILD_TMPDIR/$$INSTALL_PREFIX" \
      $$INSTALLDIR/lib/python{version}/_sysconfigdata__*.py
    rm -f $$INSTALLDIR/lib/python{version}/config-*/Makefile
    find $$INSTALLDIR/lib/python{version} -name '*.exe' -delete
    """.format(
        version = PYTHON_MAJOR_MINOR,
    ),
)

foreign_cc_shared_wrapper(
    name = "python_unix_shared",
    input = ":python_unix",
    visibility = ["//visibility:public"],
)

# A copy of python_unix with rpaths rewritten so the interpreter and its
# lib-dynload modules can find libpython and the dynamic deps at runtime.
# Use this (not :python_unix) when invoking python from a Bazel action.
foreign_cc_runnable(
    name = "python_unix_runnable",
    input = ":python_unix",
    binary_path = "bin/python{}".format(PYTHON_MAJOR_MINOR),
    patch_globs = [
        "lib/python{}/lib-dynload/*.so".format(PYTHON_MAJOR_MINOR),
    ],
    target_compatible_with = select({
        "@platforms//os:windows": ["@platforms//:incompatible"],
        "//conditions:default": [],
    }),
    visibility = ["//visibility:public"],
)

# TODO(ABLD-392): temporary workaround -- remove once the agent build runs the
# Python we ship instead of the host Python for extension module compilation.
# Python extension modules link against these shared libs and need them present
# in {install_dir}/embedded/lib before any Python invocation (e.g. pip, setuptools).
# Collected automatically by walking python_pkg's installed_files and dynamic_deps.
dd_collect_dependencies(
    name = "_runtime_deps",
    srcs = [":python_pkg"],
    visibility = ["//visibility:private"],
)

pkg_filegroup(
    name = "runtime_deps",
    srcs = [":_runtime_deps"],
    prefix = "embedded",
    visibility = ["//visibility:public"],
)

pkg_files(
    name = "install_files_win",
    srcs = [":python_win"],
    strip_prefix = "build",
)

# On Windows, python_win is built via MSBuild (not cc_shared_library), so there are no
# dynamic_deps edges for dd_collect_dependencies to walk automatically from python_pkg.
# Explicitly collect OpenSSL packaged targets so that embedded3/include/openssl/ and
# embedded3/lib/*.lib are present (required for FIPS cryptography source builds).
dd_collect_dependencies(
    name = "_openssl_deps_win",
    srcs = [
        "@openssl//:libssl_shared_pkg",
        "@openssl//:libcrypto_shared_pkg",
    ],
    visibility = ["//visibility:private"],
)

# We resort to select_file here instead of picking the output groups with filegroup
# because configure_make names its output groups by "basename", for which we have a conflict:
# both the versioned python binary as well as the library root (under lib) share the same name.
# select_file lets us be more specific and remove ambiguity
select_file(
    name = "python_lib_dir_group_unix",
    srcs = ":python_unix",
    subpath = "lib/python{}".format(PYTHON_MAJOR_MINOR),
)

select_file(
    name = "python3_bin",
    srcs = ":python_unix",
    subpath = "bin/python{}".format(PYTHON_MAJOR_MINOR),
    visibility = ["//visibility:public"],
)

# Create symlinks for libpython (rules_pkg 1.2+ supports symlinks in pkg_install)
pkg_mklink(
    name = "libpython_symlink",
    link_name = "lib/libpython{}.so".format(PYTHON_MAJOR_MINOR),
    target = LINUX_SO,
    attributes = pkg_attributes("0644"),
)

BIN_SYMLINKS = {
    "bin/python3": "python{}".format(PYTHON_MAJOR_MINOR),
    "bin/python": "python3",
}

[
    pkg_mklink(
        name = "python_bin_symlink_" + target,
        link_name = link,
        target = target,
        attributes = pkg_attributes("0755"),
    )
    for link, target in BIN_SYMLINKS.items()
]

filegroup(
    name = "headers_unix",
    srcs = [":python_unix"],
    output_group = "include",
)

filegroup(
    name = "python_lib_unix",
    srcs = [":python_unix"],
    output_group = select({
        "@platforms//os:linux": LINUX_SO,
        "@platforms//os:macos": MACOS_DYLIB,
    }),
    target_compatible_with = select({
        "@platforms//os:macos": [],
        "@platforms//os:linux": [],
        "//conditions:default": ["@platforms//:incompatible"],
    }),
)

# configure_make doesn't provide output groups for "data files", so we use select_file instead
select_file(
    name = "python_stable_lib_linux",
    srcs = ":python_unix",
    subpath = "lib/libpython3.so",
    target_compatible_with = ["@platforms//os:linux"],
)

pkg_files(
    name = "install_headers_unix",
    srcs = [":headers_unix"],
)

# Grouped helper for python_pkg installed_files: installed_files is non-configurable,
# so the Linux-only libpython_symlink select lives here instead.
pkg_filegroup(
    name = "_python_pkg_common_files",
    srcs = [
        ":install_headers_unix",
    ] + [":python_bin_symlink_" + name for name in BIN_SYMLINKS.values()] + select({
        "@platforms//os:linux": [":libpython_symlink"],
        "//conditions:default": [],
    }),
)

dd_cc_packaged(
    name = "python_pkg",
    input = ":python_unix_shared",
    installed_files = [":_python_pkg_common_files"],
    installed_executables = {
        ":python3_bin": "bin",
        ":python_lib_dir_group_unix": "lib",
    } | select({
        "@platforms//os:linux": {":python_stable_lib_linux": "lib"},
        "//conditions:default": {},
    }),
    visibility = ["//visibility:public"],
)

pkg_filegroup(
    name = "all_files",
    srcs = select({
        "@platforms//os:windows": [
            ":install_files_win",
            ":_openssl_deps_win",
        ],
        "//conditions:default": [
            ":install_headers_unix",
        ] + [":python_bin_symlink_" + name for name in BIN_SYMLINKS.values()],
    }) + select({
        "@platforms//os:linux": [":libpython_symlink"],
        "//conditions:default": [],
    }),
    prefix = select({
        "@platforms//os:windows": "embedded3",
        "//conditions:default": "embedded",
    }),
    visibility = ["//visibility:public"],
)

pkg_install(
    name = "install",
    srcs = select({
        "@platforms//os:windows": [":all_files"],
        "//conditions:default": [":runtime_deps"],
    }),
)