diff --git a/eng/ci_tools.txt b/eng/ci_tools.txt index b8ebda3d8b87..0d44ae300259 100644 --- a/eng/ci_tools.txt +++ b/eng/ci_tools.txt @@ -15,6 +15,7 @@ typing-extensions==4.12.2 pyproject-api==1.8.0 cibuildwheel==2.16.5 importlib-metadata==8.5.0 +build==1.2.2.post1 # requirements leveraged for testing pytest==8.3.5 diff --git a/eng/scripts/get_package_properties.py b/eng/scripts/get_package_properties.py index db32087db8da..c87594ab844b 100644 --- a/eng/scripts/get_package_properties.py +++ b/eng/scripts/get_package_properties.py @@ -47,7 +47,7 @@ # Use abspath for the os.walk because if setup parsing fails it often changes cwd which throws off the relative walk for root, dirs, files in os.walk(os.path.abspath(args.search_path)): if re.search(r"sdk[\\/][^\\/]+[\\/][^\\/]+$", root): - if "setup.py" in files: + if "setup.py" in files or "pyproject.toml" in files: try: parsed = ParsedSetup.from_path(root) diff --git a/eng/test_tools.txt b/eng/test_tools.txt index 9852d82005d8..62fa480c6b1d 100644 --- a/eng/test_tools.txt +++ b/eng/test_tools.txt @@ -6,7 +6,9 @@ pytest-custom-exit-code==0.3.0 pytest-xdist==3.2.1 coverage==7.6.1 bandit==1.6.2 +protobuf==3.17.3; python_version == '2.7' pyproject-api==1.8.0 +build==1.2.2.post1 # locking packages defined as deps from azure-sdk-tools Jinja2==3.1.6 diff --git a/eng/tox/install_depend_packages.py b/eng/tox/install_depend_packages.py index 398f5af61da3..2b9b5b2014d7 100644 --- a/eng/tox/install_depend_packages.py +++ b/eng/tox/install_depend_packages.py @@ -311,7 +311,7 @@ def check_req_against_exclusion(req, req_to_exclude): def filter_dev_requirements( - setup_py_path, + package_directory, released_packages, temp_dir, additional_filter_fn: Optional[Callable[[str, List[str], List[Requirement]], List[str]]] = None, @@ -324,7 +324,7 @@ def filter_dev_requirements( by the package). """ # This method returns list of requirements from dev_requirements by filtering out packages in given list - dev_req_path = os.path.join(os.path.dirname(setup_py_path), DEV_REQ_FILE) + dev_req_path = os.path.join(package_directory, DEV_REQ_FILE) requirements = [] with open(dev_req_path, "r") as dev_req_file: requirements = dev_req_file.readlines() @@ -348,7 +348,7 @@ def filter_dev_requirements( if additional_filter_fn: # this filter function handles the case where a dev requirement is incompatible with the current set of targeted packages - filtered_req = additional_filter_fn(setup_py_path, filtered_req, released_packages) + filtered_req = additional_filter_fn(package_directory, filtered_req, released_packages) logging.info("Filtered dev requirements: %s", filtered_req) @@ -410,10 +410,12 @@ def install_packages(packages, req_file): ) args = parser.parse_args() - setup_path = os.path.join(os.path.abspath(args.target_package), "setup.py") + + setup_path = os.path.join(os.path.abspath(args.target_package)) if not (os.path.exists(setup_path) and os.path.exists(args.work_dir)): logging.error("Invalid arguments. Please make sure target directory and working directory are valid path") sys.exit(1) + install_dependent_packages(setup_path, args.dependency_type, args.work_dir) diff --git a/eng/tox/tox.ini b/eng/tox/tox.ini index e3e79bb23dc8..c449ebc8994c 100644 --- a/eng/tox/tox.ini +++ b/eng/tox/tox.ini @@ -19,7 +19,6 @@ envlist = whl,sdist deps = -r {repository_root}/eng/test_tools.txt - [coverage:paths] source = azure @@ -45,6 +44,7 @@ pkgs = packaging==24.2 urllib3==2.2.3 tomli==2.2.1 + build==1.2.2.post1 [pytest] ignore_args=--ignore=.tox --ignore=build --ignore=.eggs --ignore=samples @@ -415,7 +415,7 @@ deps = {[packaging]pkgs} commands = python -m pip install {repository_root}/tools/azure-sdk-tools --no-deps - python {tox_root}/setup.py --q sdist -d {envtmpdir} + sdk_build_package --package_folder {tox_root} -d {envtmpdir} --package_type sdist python {repository_root}/eng/tox/verify_sdist.py -d {envtmpdir} -t {tox_root} diff --git a/eng/tox/verify_sdist.py b/eng/tox/verify_sdist.py index 5f7c45756491..4392b33a7613 100644 --- a/eng/tox/verify_sdist.py +++ b/eng/tox/verify_sdist.py @@ -42,7 +42,7 @@ def get_root_directories_in_sdist(dist_dir: str, version: str) -> List[str]: """ # find sdist zip file # extract sdist and find list of directories in sdist - path_to_zip = glob.glob(os.path.join(dist_dir, "*{}*.tar.gz".format(version)))[0] + path_to_zip = glob.glob(os.path.join(dist_dir, "**", "*{}*.tar.gz".format(version)), recursive=True)[0] extract_location = os.path.join(dist_dir, "unzipped") # Cleanup any files in unzipped cleanup(extract_location) diff --git a/sdk/keyvault/azure-keyvault-keys/pyproject.toml b/sdk/keyvault/azure-keyvault-keys/pyproject.toml index e00361912969..30f48984a99b 100644 --- a/sdk/keyvault/azure-keyvault-keys/pyproject.toml +++ b/sdk/keyvault/azure-keyvault-keys/pyproject.toml @@ -1,2 +1,49 @@ +[build-system] +requires = ["setuptools>=61.0.0", "wheel"] # Requires 61.0.0 for dynamic version +build-backend = "setuptools.build_meta" + +[project] +name = "azure-keyvault-keys" +authors = [ + {name = "Microsoft Corporation", email = "azurekeyvault@microsoft.com"}, +] +description = "Microsoft Azure Key Vault Keys Client Library for Python" +keywords = ["azure", "azure sdk"] +requires-python = ">=3.8" +license = {text = "MIT License"} +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "License :: OSI Approved :: MIT License", +] +dependencies = [ + "azure-core>=1.31.0", + "cryptography>=2.1.4", + "isodate>=0.6.1", + "typing-extensions>=4.6.0", +] +dynamic = ["version", "readme"] + +[project.urls] +repository = "https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/keyvault/azure-keyvault-keys" + +[tool.setuptools.dynamic] +version = {attr = "azure.keyvault.keys._version.VERSION"} +readme = {file = ["README.md", "CHANGELOG.md"], content-type = "text/markdown"} + +[tool.setuptools.packages.find] +exclude = ["samples*", "tests*", "azure", "azure.keyvault"] + +[tool.setuptools.package-data] +pytyped = ["py.typed"] + [tool.azure-sdk-build] pyright = false diff --git a/sdk/keyvault/azure-keyvault-keys/setup.py b/sdk/keyvault/azure-keyvault-keys/setup.py deleted file mode 100644 index 276ae663cf6b..000000000000 --- a/sdk/keyvault/azure-keyvault-keys/setup.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python - -# ------------------------------------ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# ------------------------------------ -# pylint:disable=missing-docstring - -import re -import os.path -from io import open -from setuptools import find_packages, setup - -# Change the PACKAGE_NAME only to change folder and different name -PACKAGE_NAME = "azure-keyvault-keys" -PACKAGE_PPRINT_NAME = "Key Vault Keys" - -# a-b-c => a/b/c -PACKAGE_FOLDER_PATH = PACKAGE_NAME.replace("-", "/") -# a-b-c => a.b.c -NAMESPACE_NAME = PACKAGE_NAME.replace("-", ".") - -# Version extraction inspired from 'requests' -with open(os.path.join(PACKAGE_FOLDER_PATH, "_version.py"), "r") as fd: - VERSION = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1) - -if not VERSION: - raise RuntimeError("Cannot find version information") - -with open("README.md", encoding="utf-8") as f: - README = f.read() -with open("CHANGELOG.md", encoding="utf-8") as f: - CHANGELOG = f.read() - -setup( - name=PACKAGE_NAME, - version=VERSION, - include_package_data=True, - description=f"Microsoft Azure {PACKAGE_PPRINT_NAME} Client Library for Python", - long_description=README + "\n\n" + CHANGELOG, - long_description_content_type="text/markdown", - license="MIT License", - author="Microsoft Corporation", - author_email="azurekeyvault@microsoft.com", - url="https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/keyvault/azure-keyvault-keys", - keywords="azure, azure sdk", - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Programming Language :: Python", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "License :: OSI Approved :: MIT License", - ], - zip_safe=False, - packages=find_packages( - exclude=[ - "samples", - "tests", - # Exclude packages that will be covered by PEP420 or nspkg - "azure", - "azure.keyvault", - ] - ), - python_requires=">=3.8", - install_requires=[ - "azure-core>=1.31.0", - "cryptography>=2.1.4", - "isodate>=0.6.1", - "typing-extensions>=4.6.0", - ], -) diff --git a/tools/azure-sdk-tools/ci_tools/build.py b/tools/azure-sdk-tools/ci_tools/build.py index e08055846153..e4e17a5ad491 100644 --- a/tools/azure-sdk-tools/ci_tools/build.py +++ b/tools/azure-sdk-tools/ci_tools/build.py @@ -10,6 +10,57 @@ from ci_tools.versioning.version_set_dev import get_dev_version, format_build_id +def build_package() -> None: + parser = argparse.ArgumentParser( + description="""This is a secondary entrypoint for the "build" action. This command is used to install dependencies and build a specific package within the azure-sdk-for-python repository.""", + ) + + parser.add_argument( + "-d", + "--distribution-directory", + dest="distribution_directory", + help="The path to the distribution directory. Should be passed $(Build.ArtifactStagingDirectory) from the devops yaml definition." + + "If that is not provided, will default to env variable SDK_ARTIFACT_DIRECTORY -> /.artifacts.", + ) + + parser.add_argument( + "--package_folder", + dest="package_folder", + required=True, + help="The target that should be assembled", + ) + + parser.add_argument( + "--package_type", + dest="package_type", + choices=["sdist", "whl", "all"], + default="all", + help="The type of package to build: sdist (source distribution only), whl (wheel only), or all (both)", + ) + + args = parser.parse_args() + + target_package = ParsedSetup.from_path(args.package_folder) + artifact_directory = get_artifact_directory(args.distribution_directory) + + enable_sdist = True + enable_wheel = True + + if args.package_type == "sdist": + enable_wheel = False + + if args.package_type == "whl": + enable_sdist = False + + build_packages( + [target_package.folder], + artifact_directory, + False, + DEFAULT_BUILD_ID, + enable_wheel, + enable_sdist + ) + def build() -> None: parser = argparse.ArgumentParser( description="""This is the primary entrypoint for the "build" action. This command is used to build any package within the azure-sdk-for-python repository.""", @@ -125,6 +176,8 @@ def build() -> None: artifact_directory, str_to_bool(args.is_dev_build), build_id, + True, + True ) @@ -146,6 +199,8 @@ def build_packages( distribution_directory: Optional[str] = None, is_dev_build: bool = False, build_id: str = "", + enable_wheel: bool = True, + enable_sdist: bool = True ): logging.log(level=logging.INFO, msg=f"Generating {targeted_packages} using python{sys.version}") @@ -168,7 +223,7 @@ def build_packages( set_version_py(setup_parsed.setup_filename, new_version) set_dev_classifier(setup_parsed.setup_filename, new_version) - create_package(package_root, dist_dir) + create_package(package_root, dist_dir, enable_wheel, enable_sdist) def create_package( @@ -193,7 +248,7 @@ def create_package( pip_output = get_pip_list_output(sys.executable) necessary_install_requirements = [req for req in setup_parsed.requires if parse_require(req).key not in pip_output.keys()] run([sys.executable, "-m", "pip", "install", *necessary_install_requirements], cwd=setup_parsed.folder) - run([sys.executable, "-m", "build", f"-n{'s' if enable_sdist else ''}{'w' if enable_wheel else ''}", "-o", dist], cwd=setup_parsed.folder) + run([sys.executable, "-m", "build", f"-n{'s' if enable_sdist else ''}{'w' if enable_wheel else ''}", "-o", dist], cwd=setup_parsed.folder, check=True) else: if enable_wheel: if setup_parsed.ext_modules: diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index 0daa47eba12c..ed1309314824 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -406,7 +406,7 @@ def process_requires(setup_py_path: str, is_dev_build: bool = False): else: logging.info("Packages not available on PyPI:{}".format(requirement_to_update)) update_requires(setup_py_path, requirement_to_update) - logging.info("Package requirement is updated in setup.py") + logging.info(f"Package requirement is updated in {'pyproject.toml' if pkg_details.is_pyproject else 'setup.py'}.") def find_sdist(dist_dir: str, pkg_name: str, pkg_version: str) -> Optional[str]: diff --git a/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py b/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py index 866c8a12f892..638b3c2f8208 100644 --- a/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py +++ b/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py @@ -390,6 +390,7 @@ def parse_pyproject( python_requires = project_config.get("requires-python") requires = project_config.get("dependencies") is_new_sdk = name in NEW_REQ_PACKAGES or any(map(lambda x: (parse_require(x).key in NEW_REQ_PACKAGES), requires)) + # todo: update traversal logic name_space = name.replace("-", ".") package_data = get_value_from_dict(toml_dict, "tool.setuptools.package-data", None) include_package_data = get_value_from_dict(toml_dict, "tool.setuptools.include-package-data", True) diff --git a/tools/azure-sdk-tools/ci_tools/scenario/generation.py b/tools/azure-sdk-tools/ci_tools/scenario/generation.py index 237511cedca5..4487e28bb674 100644 --- a/tools/azure-sdk-tools/ci_tools/scenario/generation.py +++ b/tools/azure-sdk-tools/ci_tools/scenario/generation.py @@ -51,7 +51,6 @@ def create_package_and_install( commands_options = [] built_pkg_path = "" - setup_py_path = os.path.join(target_setup, "setup.py") additional_downloaded_reqs = [] if not os.path.exists(distribution_directory): @@ -69,10 +68,10 @@ def create_package_and_install( commands_options.extend(["--cache-dir", cache_dir]) discovered_packages = discover_packages( - setup_py_path, distribution_directory, target_setup, package_type, force_create + target_setup, distribution_directory, target_setup, package_type, force_create ) - target_package = ParsedSetup.from_path(setup_py_path) + target_package = ParsedSetup.from_path(target_setup) # ensure that discovered packages are always copied to the distribution directory regardless of other factors for built_package in discovered_packages: @@ -104,7 +103,7 @@ def create_package_and_install( logging.info("Installing {w} from fresh built package.".format(w=built_package)) if not pre_download_disabled: - requirements = ParsedSetup.from_path(os.path.join(os.path.abspath(target_setup), "setup.py")).requires + requirements = ParsedSetup.from_path(os.path.join(os.path.abspath(target_setup))).requires azure_requirements = [req.split(";")[0] for req in requirements if req.startswith("azure-")] if azure_requirements: diff --git a/tools/azure-sdk-tools/setup.py b/tools/azure-sdk-tools/setup.py index 73e6586c3838..2ab37373ac01 100644 --- a/tools/azure-sdk-tools/setup.py +++ b/tools/azure-sdk-tools/setup.py @@ -43,6 +43,7 @@ "sdk_generator=packaging_tools.sdk_generator:generate_main", "sdk_package=packaging_tools.sdk_package:generate_main", "sdk_build=ci_tools.build:build", + "sdk_build_package=ci_tools.build:build_package", "sdk_build_conda=ci_tools.conda:entrypoint", "sdk_set_dev_version=ci_tools.versioning.version_set_dev:version_set_dev_main", "sdk_set_version=ci_tools.versioning.version_set:version_set_main",