diff --git a/CLAUDE.md b/CLAUDE.md index 7d57b4a52..6a1f0d8d7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ This is the **Calunga / Trusted Libraries Index** — a Red Hat project that mai ## Key Concepts -- **onboarded_packages/**: The source of truth — one JSON file per package with `{"version": "...", "ignored_versions": [...]}`. The `version` field specifies the latest version to build; `ignored_versions` lists versions to skip during automated updates. An optional `build_extra` field (list of bare package names) declares undeclared dependencies that must be built alongside the package — `identify-packages` resolves each to its current version from the corresponding JSON file. Every entry in `build_extra` must have a matching onboarded package JSON or the build will fail. +- **onboarded_packages/**: The source of truth — one JSON file per package with `{"version": "...", "ignored_versions": [...]}`. The `version` field specifies the latest version to build; `ignored_versions` lists versions to skip during automated updates. An optional `build_extra` field (list of bare package names) declares undeclared dependencies that must be built alongside the package — `identify-packages` resolves each to its current version from the corresponding JSON file. Every entry in `build_extra` must have a matching onboarded package JSON or the build will fail. An optional `sdist_url` field (e.g. `"git+https://github.com/org/repo.git"`) directs the build to fetch the source from a git URL instead of PyPI — `identify-packages` produces a PEP 440 URL requirement (`pkg @ url@version`) when this field is present. - **Build pipeline**: Tekton/Konflux on OpenShift. Builds wheels from source, runs security scans (Snyk, Coverity, ClamAV, SAST), and pushes OCI artifacts to Quay - **Automated updates**: GitHub Actions workflow checks PyPI for new versions not yet in Pulp, creates PRs with auto-merge diff --git a/hack/identify-packages b/hack/identify-packages index 294bd0aca..48f9c7e62 100755 --- a/hack/identify-packages +++ b/hack/identify-packages @@ -16,6 +16,13 @@ extract_version() { grep -o '"version": *"[^"]*"' | sed 's/.*"version": *"\([^"]*\)".*/\1/' || true } +# Extract the "sdist_url" value from a JSON file using grep/sed. +# Returns empty string if the field is absent. +# jq not available in the Tekton SCRIPT_RUNNER_IMAGE +extract_sdist_url() { + grep -o '"sdist_url": *"[^"]*"' | sed 's/.*"sdist_url": *"\([^"]*\)".*/\1/' || true +} + # Extract package names from the build_extra JSON array. # Handles both single-line and multi-line formatted arrays. # jq not available in the Tekton SCRIPT_RUNNER_IMAGE @@ -49,7 +56,12 @@ for f in $CHANGED_FILES; do OLD_VERSION="$(git show "${REVISION}:${f}" 2>/dev/null | extract_version || true)" if [[ "$CURRENT_VERSION" != "$OLD_VERSION" ]]; then - NEW_PACKAGES="${NEW_PACKAGES}${PKG_NAME}==${CURRENT_VERSION}"$'\n' + SDIST_URL="$(extract_sdist_url < "$f")" + if [[ -n "$SDIST_URL" ]]; then + NEW_PACKAGES="${NEW_PACKAGES}${PKG_NAME} @ ${SDIST_URL}@${CURRENT_VERSION}"$'\n' + else + NEW_PACKAGES="${NEW_PACKAGES}${PKG_NAME}==${CURRENT_VERSION}"$'\n' + fi fi fi done @@ -57,8 +69,13 @@ NEW_PACKAGES="$(echo -n "$NEW_PACKAGES" | sed '/^$/d')" # Resolve build_extra dependencies for each package to be built EXTRA_PACKAGES="" -for pkg_entry in $NEW_PACKAGES; do - pkg_name="${pkg_entry%%==*}" +while IFS= read -r pkg_entry; do + [[ -z "$pkg_entry" ]] && continue + if [[ "$pkg_entry" == *" @ "* ]]; then + pkg_name="${pkg_entry%% @*}" + else + pkg_name="${pkg_entry%%==*}" + fi pkg_file="${PACKAGES_DIR}/${pkg_name}.json" if [[ -f "$pkg_file" ]]; then extras="$(extract_build_extra < "$pkg_file")" || true @@ -73,14 +90,19 @@ for pkg_entry in $NEW_PACKAGES; do echo "[ERROR] Could not extract version from ${dep_file}" >&2 exit 1 fi - dep_entry="${dep}==${dep_version}" - if ! echo "$NEW_PACKAGES" | grep -qx "$dep_entry" && \ - ! echo "$EXTRA_PACKAGES" | grep -qx "$dep_entry"; then + dep_sdist_url="$(extract_sdist_url < "$dep_file")" + if [[ -n "$dep_sdist_url" ]]; then + dep_entry="${dep} @ ${dep_sdist_url}@${dep_version}" + else + dep_entry="${dep}==${dep_version}" + fi + if ! echo "$NEW_PACKAGES" | grep -Fqx "$dep_entry" && \ + ! echo "$EXTRA_PACKAGES" | grep -Fqx "$dep_entry"; then EXTRA_PACKAGES="${EXTRA_PACKAGES}${dep_entry}"$'\n' fi done fi -done +done <<< "$NEW_PACKAGES" EXTRA_PACKAGES="$(echo -n "$EXTRA_PACKAGES" | sed '/^$/d')" if [[ -n "$EXTRA_PACKAGES" ]]; then echo "Extra dependencies to build:" >&2 diff --git a/onboarded_packages/griffe.json b/onboarded_packages/griffe.json new file mode 100644 index 000000000..da107b093 --- /dev/null +++ b/onboarded_packages/griffe.json @@ -0,0 +1,157 @@ +{ + "version": "2.1.0", + "sdist_url": "git+https://github.com/mkdocstrings/griffe", + "ignored_versions": [ + "0.1.0", + "0.2.0", + "0.3.0", + "0.4.0", + "0.5.0", + "0.6.0", + "0.7.0", + "0.7.1", + "0.8.0", + "0.9.0", + "0.10.0", + "0.11.0", + "0.11.1", + "0.11.2", + "0.11.3", + "0.11.4", + "0.11.5", + "0.11.6", + "0.11.7", + "0.12.0", + "0.12.1", + "0.12.2", + "0.12.3", + "0.12.4", + "0.12.5", + "0.12.6", + "0.13.0", + "0.13.1", + "0.13.2", + "0.14.0", + "0.14.1", + "0.15.0", + "0.15.1", + "0.16.0", + "0.17.0", + "0.18.0", + "0.19.0", + "0.19.1", + "0.19.2", + "0.19.3", + "0.20.0", + "0.21.0", + "0.22.0", + "0.22.1", + "0.22.2", + "0.23.0", + "0.24.0", + "0.24.1", + "0.25.0", + "0.25.1", + "0.25.2", + "0.25.3", + "0.25.4", + "0.25.5", + "0.26.0", + "0.27.0", + "0.27.1", + "0.27.2", + "0.27.3", + "0.27.4", + "0.27.5", + "0.28.0", + "0.28.1", + "0.28.2", + "0.29.0", + "0.29.1", + "0.30.0", + "0.30.1", + "0.31.0", + "0.32.0", + "0.32.1", + "0.32.2", + "0.32.3", + "0.33.0", + "0.34.0", + "0.35.0", + "0.35.1", + "0.35.2", + "0.36.0", + "0.36.1", + "0.36.2", + "0.36.3", + "0.36.4", + "0.36.5", + "0.36.6", + "0.36.7", + "0.36.8", + "0.36.9", + "0.37.0", + "0.38.0", + "0.38.1", + "0.39.0", + "0.39.1", + "0.40.0", + "0.40.1", + "0.41.0", + "0.41.1", + "0.41.2", + "0.41.3", + "0.42.0", + "0.42.1", + "0.42.2", + "0.43.0", + "0.44.0", + "0.45.0", + "0.45.1", + "0.45.2", + "0.45.3", + "0.46.0", + "0.46.1", + "0.47.0", + "0.48.0", + "0.49.0", + "1.0.0", + "1.1.0", + "1.1.1", + "1.2.0", + "1.3.0", + "1.3.1", + "1.3.2", + "1.4.0", + "1.4.1", + "1.5.0", + "1.5.1", + "1.5.2", + "1.5.3", + "1.5.4", + "1.5.5", + "1.5.6", + "1.5.7", + "1.6.0", + "1.6.1", + "1.6.2", + "1.6.3", + "1.7.0", + "1.7.1", + "1.7.2", + "1.7.3", + "1.8.0", + "1.9.0", + "1.10.0", + "1.11.0", + "1.11.1", + "1.12.0", + "1.12.1", + "1.13.0", + "1.14.0", + "1.15.0", + "2.0.0", + "2.0.1", + "2.0.2" + ] +}