Skip to content

Commit da55044

Browse files
authored
add support for conda packages (#204)
1 parent 7fb950b commit da55044

21 files changed

+415
-16
lines changed

.github/ISSUE_TEMPLATE/new-check.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,9 @@ body:
5050
label: Distribution type
5151
description: What types of distribution does this check apply to?
5252
options:
53-
- label: source (e.g. `.tar.gz`)
53+
- label: source (e.g. `.tar.gz`, `.zip`)
5454
required: false
55-
- label: built (e.g. `.whl`)
55+
- label: built (e.g. `.conda`, `.tar.bz2`, `.whl`)
5656
required: false
5757
- type: textarea
5858
attributes:

.gitignore

+4
Original file line numberDiff line numberDiff line change
@@ -66,12 +66,16 @@ wheels/
6666
*.zstd
6767

6868
# exclusions
69+
!tests/data/osx-64-baseballmetrics-0.1.0-0.conda
70+
!tests/data/osx-64-baseballmetrics-0.1.0-0.tar.bz2
6971
!tests/data/baseballmetrics-0.1.0-py3-none-macosx_10_15_x86_64.macosx_11_6_x86_64.macosx_12_5_x86_64.whl
7072
!tests/data/baseballmetrics-0.1.0-py3-none-manylinux_2_28_x86_64.manylinux_2_5_x86_64.manylinux1_x86_64.whl
7173
!tests/data/debug-baseballmetrics-0.1.0-macosx-wheel.tar.bz2
7274
!tests/data/debug-baseballmetrics-0.1.0-macosx-wheel.tar.gz
7375
!tests/data/debug-baseballmetrics-0.1.0-py3-none-macosx_10_15_x86_64.macosx_11_6_x86_64.macosx_12_5_x86_64.whl
7476
!tests/data/debug-baseballmetrics-py3-none-manylinux_2_28_x86_64.manylinux_2_5_x86_64.manylinux1_x86_64.whl
77+
!tests/data/osx-64-debug-baseballmetrics-0.1.0-0.conda
78+
!tests/data/osx-64-debug-baseballmetrics-0.1.0-0.tar.bz2
7579
!tests/data/base-package-0.1.0.tar.gz
7680
!tests/data/base-package-0.1.0.zip
7781
!tests/data/lightgbm-3.3.5-py3-none-win_amd64.whl

Makefile

+17-3
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ format:
4040

4141
.PHONY: install
4242
install:
43-
pipx install --force .
43+
pipx install --force '.[conda]'
4444

4545
.PHONY: lint
4646
lint:
@@ -60,7 +60,7 @@ lint:
6060
mypy ./tests/data
6161
yamllint \
6262
--strict \
63-
-d '{extends: default, rules: {truthy: {check-keys: false}, line-length: {max: 120}}}' \
63+
-d '{extends: default, rules: {braces: {max-spaces-inside: 1}, truthy: {check-keys: false}, line-length: {max: 120}}}' \
6464
.
6565

6666
.PHONY: linux-wheel
@@ -94,7 +94,21 @@ test-data-sdist:
9494
.PHONY: test-data-bdist
9595
test-data-bdist: \
9696
linux-wheel \
97-
mac-wheel
97+
mac-wheel \
98+
test-data-macos-conda-tarbz2-packages \
99+
test-data-conda-dot-conda-packages
100+
101+
# NOTE: .bz2 packages were created with conda-build 3.27.0
102+
.PHONY: test-data-conda-packages
103+
test-data-macos-conda-tarbz2-packages:
104+
bin/create-test-data-conda.sh 'osx-64'
105+
106+
.PHONY: test-data-conda-dot-conda-packages
107+
test-data-conda-dot-conda-packages:
108+
cph transmute \
109+
--out-folder ./tests/data \
110+
'./tests/data/*-0.tar.bz2' \
111+
'.conda'
98112

99113
.PHONY: test
100114
test:

README.md

+7
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,13 @@
1717

1818
It's inspired by R's `R CMD check`.
1919

20+
Supported formats:
21+
22+
* Python sdists
23+
* Python wheels
24+
* `conda` packages (both `.conda` and `.tar.bz2`)
25+
* any `.tar.bz2`, `.tar.gz`, or `.zip` archive
26+
2027
See ["How to Test a Python Distribution"](https://pydistcheck.readthedocs.io/en/latest/how-to-test-a-python-distribution.html) to see how it and similar tools like [`auditwheel`](https://github.com/pypa/auditwheel), [`check-wheel-contents`](https://github.com/jwodder/check-wheel-contents), and [`twine check`](https://twine.readthedocs.io/en/stable/#twine-check) fit into Python development workflows.
2128

2229
For more background on the value of such a tool, see the SciPy 2022 talk "Does that CSV Belong on PyPI? Probably Not" ([video link](https://www.youtube.com/watch?v=1a7g5l_g_U8)).

bin/create-test-data-conda.sh

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#!/bin/bash
2+
3+
# [description]
4+
#
5+
# generate test conda packages
6+
#
7+
# [references]
8+
#
9+
# - https://conda.io/projects/conda/en/latest/user-guide/configuration/use-condarc.html
10+
# - https://github.com/conda/conda-build/blame/main/docs/source/resources/package-spec.rst
11+
# - https://github.com/conda/conda-docs/issues/796
12+
# - https://docs.anaconda.com/free/anacondaorg/user-guide/packages/manage-packages/#conda-compression-format
13+
14+
CHANNEL="${1}"
15+
16+
set -Eeuox pipefail
17+
18+
REPO_ROOT="${PWD}"
19+
TEST_DATA_DIR="${PWD}/tests/data"
20+
CONDA_BASE_DIR=$(
21+
conda info --base
22+
)
23+
24+
new_build_dir() {
25+
rm -rf ./conda-build
26+
mkdir ./conda-build
27+
cd ./conda-build
28+
}
29+
30+
conda_build() {
31+
cd "${REPO_ROOT}"
32+
new_build_dir
33+
conda build \
34+
--no-anaconda-upload \
35+
--no-test \
36+
--no-verify \
37+
--old-build-string \
38+
"${1}"
39+
rm -f ./build_env_setup.sh
40+
}
41+
42+
# start with .tar.bz2 packages
43+
conda config --set conda_build.pkg_format 1
44+
45+
#----------------------------#
46+
#- baseballmetrics*.tar.bz2 -#
47+
#----------------------------#
48+
conda_build ../tests/data/conda-recipes/baseballmetrics
49+
50+
#----------------------------------#
51+
#- debug-baseballmetrics*.tar.bz2 -#
52+
#----------------------------------#
53+
conda_build ../tests/data/conda-recipes/debug-baseballmetrics
54+
55+
# get packages
56+
cp \
57+
"${CONDA_BASE_DIR}/conda-bld/${CHANNEL}/baseballmetrics-0.1.0-0.tar.bz2" \
58+
"${TEST_DATA_DIR}/${CHANNEL}-baseballmetrics-0.1.0-0.tar.bz2"
59+
60+
cp \
61+
"${CONDA_BASE_DIR}/conda-bld/${CHANNEL}/debug-baseballmetrics-0.1.0-0.tar.bz2" \
62+
"${TEST_DATA_DIR}/${CHANNEL}-debug-baseballmetrics-0.1.0-0.tar.bz2"
63+
64+
# clean up
65+
cd "${REPO_ROOT}"
66+
rm -rf ./conda-build

bin/get-conda-release-files.py

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
import os
2+
import sys
3+
from collections import defaultdict
4+
from dataclasses import dataclass
5+
6+
import requests
7+
8+
PACKAGE_NAME = sys.argv[1]
9+
CONDA_CHANNEL = sys.argv[2]
10+
OUTPUT_DIR = sys.argv[3]
11+
BASE_URL = f"https://api.anaconda.org/package/{CONDA_CHANNEL}"
12+
13+
print(f"Getting conda-forge details for package '{PACKAGE_NAME}'")
14+
res = requests.get(url=f"{BASE_URL}/{PACKAGE_NAME}", timeout=30)
15+
res.raise_for_status()
16+
release_info = res.json()
17+
18+
latest_version = release_info["versions"][-1]
19+
files = [f for f in release_info["files"] if f["version"] == latest_version]
20+
21+
22+
@dataclass
23+
class _ReleaseFile:
24+
filename: str
25+
package_type: str
26+
url: str
27+
28+
29+
files_by_type = defaultdict(list)
30+
for file_info in files:
31+
pkg_type = file_info["attrs"]["target-triplet"]
32+
url = file_info["download_url"]
33+
if url.startswith("//"):
34+
url = f"https:{url}"
35+
files_by_type[pkg_type].append(
36+
_ReleaseFile(
37+
filename=file_info["basename"].replace("/", "-"),
38+
package_type=pkg_type,
39+
url=url,
40+
)
41+
)
42+
43+
print(f"Found the following file types for '{PACKAGE_NAME}=={latest_version}':")
44+
for file_type, release_files in files_by_type.items():
45+
print(f" * {file_type} ({len(release_files)})")
46+
47+
48+
for file_type in files_by_type:
49+
sample_release = files_by_type[file_type][0]
50+
output_file = os.path.join(OUTPUT_DIR, sample_release.filename)
51+
print(f"Downloading '{sample_release.filename}'")
52+
res = requests.get(
53+
url=sample_release.url, headers={"Accept": "application/octet-stream"}, timeout=30
54+
)
55+
res.raise_for_status()
56+
with open(output_file, "wb") as f:
57+
f.write(res.content)
58+
59+
print(f"Done downloading files into '{OUTPUT_DIR}'")

bin/run-smoke-tests.sh

+20-1
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,33 @@ echo "running smoke tests"
77
get-files() {
88
pkg_name=$1
99
rm -rf ./smoke-tests
10-
mkdir ./smoke-tests
10+
mkdir -p ./smoke-tests
1111
echo ""
1212
python bin/get-pypi-files.py \
1313
"${pkg_name}" \
1414
./smoke-tests
1515
}
1616

17+
get-conda-forge-files() {
18+
pkg_name=$1
19+
mkdir -p ./smoke-tests
20+
echo ""
21+
python bin/get-conda-release-files.py \
22+
"${pkg_name}" \
23+
'conda-forge' \
24+
./smoke-tests
25+
}
26+
27+
# conda package where conda-forge only has the old .tar.bz2 format
28+
get-conda-forge-files librmm
29+
get-conda-forge-files rmm
30+
pydistcheck \
31+
--ignore 'compiled-objects-have-debug-symbols' \
32+
./smoke-tests/*
33+
1734
# wheel-only packages
1835
get-files catboost
36+
get-conda-forge-files catboost
1937
pydistcheck \
2038
--ignore 'compiled-objects-have-debug-symbols,mixed-file-extensions,too-many-files,unexpected-files' \
2139
--max-allowed-size-compressed '100M' \
@@ -29,6 +47,7 @@ pydistcheck \
2947

3048
# package where source distro is a .zip
3149
get-files numpy
50+
get-conda-forge-files numpy
3251
pydistcheck \
3352
--ignore 'compiled-objects-have-debug-symbols,mixed-file-extensions,path-contains-spaces,unexpected-files' \
3453
--max-allowed-files 7500 \

docs/installation.rst

+15-1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,13 @@ Because of this, the preferred way to install it from PyPI is with ``pipx`` (`do
99
1010
pipx install pydistcheck
1111
12+
Checking ``.conda``-format ``conda`` packages requires some additional dependencies.
13+
To install those, run the following.
14+
15+
.. code-block:: shell
16+
17+
pipx install 'pydistcheck[conda]'
18+
1219
If that doesn't work for you, see the sections below for other options.
1320

1421
PyPI
@@ -20,6 +27,13 @@ If you do not want to use ``pipx`` but want to install from PyPI, install with `
2027
2128
pip install pydistcheck
2229
30+
Checking ``.conda``-format ``conda`` packages requires some additional dependencies.
31+
To install those, run the following.
32+
33+
.. code-block:: shell
34+
35+
pip install 'pydistcheck[conda]'
36+
2337
conda-forge
2438
***********
2539

@@ -44,4 +58,4 @@ To install the latest development (not released) version of ``pydistcheck``, clo
4458
4559
git clone https://github.com/jameslamb/pydistcheck.git
4660
cd pydistcheck
47-
pipx install .
61+
pipx install '.[conda]'

pyproject.toml

+9-2
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,11 @@ version = {attr = "pydistcheck.__version__"}
4949
[project.scripts]
5050
pydistcheck = "pydistcheck.cli:check"
5151

52+
[project.optional-dependencies]
53+
conda = [
54+
"zstandard>=0.22.0"
55+
]
56+
5257
[project.urls]
5358
homepage = "https://pydistcheck.readthedocs.io/en/latest/"
5459
documentation = "https://pydistcheck.readthedocs.io/en/latest/"
@@ -119,13 +124,15 @@ select = [
119124
]
120125

121126
[tool.ruff.lint.per-file-ignores]
122-
"src/pydistcheck/*" = [
127+
"src/pydistcheck/*.py" = [
123128
# uses of tarfile.extractall()
124129
"S202"
125130
]
126131
"src/pydistcheck/cli.py" = [
127132
# Too many branches
128-
"PLR0912"
133+
"PLR0912",
134+
# Too many statements
135+
"PLR0915"
129136
]
130137
"tests/*" = [
131138
# (flake8-bugbear) Found useless expression

requirements-tests.txt

+1
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@ pytest-cov
55
requests
66
twine
77
types-requests
8+
zstandard

src/pydistcheck/_compat.py

+17-1
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,25 @@
33
with a wide range of dependency versions.
44
"""
55

6+
from typing import Any
7+
68
try:
79
import tomllib
810
except ModuleNotFoundError:
911
import tomli as tomllib # type: ignore[no-redef]
1012

11-
__all__ = ["tomllib"]
13+
14+
def _import_zstandard() -> Any:
15+
try:
16+
import zstandard
17+
18+
return zstandard
19+
except ModuleNotFoundError as err:
20+
err_msg = (
21+
"Checking zstd-compressed files requires the 'zstandard' library. "
22+
"Install it with e.g. 'pip install zstandard'."
23+
)
24+
raise ModuleNotFoundError(err_msg) from err
25+
26+
27+
__all__ = ["tomllib", "_import_zstandard"]

0 commit comments

Comments
 (0)