[pull] master from DataDog:master

.builders/scripts/build_wheels.py

@@ -2,6 +2,7 @@
 
 import argparse
 import email.message
+import fnmatch
 import json
 import os
 import re
@@ -171,52 +172,23 @@ def wheel_was_built(wheel: Path) -> bool:
     return file_hash != wheel_hashes[wheel.name]
 
 
-def remove_test_files(wheel_path: Path) -> bool:
-    '''
-    Unpack the wheel, remove excluded test files, then repack it to rebuild RECORD correctly.
-    '''
-    # First, check whether the wheel contains any files that should be excluded. If not, leave it untouched.
-    with ZipFile(wheel_path, 'r') as zf:
-        excluded_members = [name for name in zf.namelist() if is_excluded_from_wheel(name)]
-
-    if not excluded_members:
-        # Nothing to strip, so skip rewriting the wheel
-        return False
-    with TemporaryDirectory() as td:
-        td_path = Path(td)
-
-        # Unpack the wheel into temp dir
-        unpack_wheel(wheel_path, dest_dir=td_path)
-        unpacked_dir = next(td_path.iterdir())
-        # Remove excluded files/folders
-        for root, dirs, files in unpacked_dir.walk(top_down=False):
-            for d in list(dirs):
-                full_dir = root / d
-                rel = full_dir.relative_to(unpacked_dir).as_posix()
-                if is_excluded_from_wheel(rel):
-                    shutil.rmtree(full_dir)
-                    dirs.remove(d)
-            for f in files:
-                rel = (root / f).relative_to(unpacked_dir).as_posix()
-                if is_excluded_from_wheel(rel):
-                    (root / f).unlink()
-
-        print(f'Tests removed from {wheel_path.name}')
-
-        dest_dir = wheel_path.parent
-        before = {p.resolve() for p in dest_dir.glob("*.whl")}
-        # Repack to same directory, regenerating RECORD
-        pack_wheel(unpacked_dir, dest_dir=dest_dir)
-
-        # The wheel might not be platform-specific, so repacking restores its original name.
-        # We need to move the repacked wheel to wheel_path, which was changed to be platform-specific.
-        after = {p.resolve() for p in wheel_path.parent.glob("*.whl")}
-        new_files = sorted(after - before, key=lambda p: p.stat().st_mtime, reverse=True)
-
-        if new_files:
-            shutil.move(str(new_files[0]), str(wheel_path))
-
-    return True
+def _remove_test_files_from_dir(unpacked_dir: Path) -> bool:
+    """Remove excluded test files and directories from an unpacked wheel directory."""
+    removed_any = False
+    for root, dirs, files in unpacked_dir.walk(top_down=False):
+        for d in list(dirs):
+            full_dir = root / d
+            rel = full_dir.relative_to(unpacked_dir).as_posix()
+            if is_excluded_from_wheel(rel):
+                shutil.rmtree(full_dir)
+                dirs.remove(d)
+                removed_any = True
+        for f in files:
+            rel = (root / f).relative_to(unpacked_dir).as_posix()
+            if is_excluded_from_wheel(rel):
+                (root / f).unlink()
+                removed_any = True
+    return removed_any
 
 
 @cache
@@ -249,6 +221,122 @@ def is_excluded_from_wheel(path: str | Path) -> bool:
     return False
 
 
+@cache
+def _load_line_removal_rules() -> list[dict]:
+    '''
+    Load the line removal rules from the toml file and compile the regex patterns.
+    '''
+
+    config_p = Path(__file__).parent / "lines_to_remove.toml"
+    with open(config_p, "rb") as f:
+        config = tomllib.load(f)
+
+    rules = config.get("rules", [])
+    for rule in rules:
+        rule['_compiled_patterns'] = [re.compile(p) for p in rule.get('line_patterns', [])]
+    return rules
+
+
+def _get_matching_rules(wheel_name: str) -> list[dict]:
+    '''
+    Match the rules in the toml to the wheel name.
+    '''
+    rules = _load_line_removal_rules()
+    matching = []
+    for rule in rules:
+        pattern = rule.get('package_pattern', '*')
+        if fnmatch.fnmatch(wheel_name, pattern):
+            matching.append(rule)
+    return matching
+
+
+def _strip_lines_from_dir(unpacked_dir: Path, rules: list[dict]) -> bool:
+    """Strip matching lines and their indented blocks from files."""
+    modified = False
+    for rule in rules:
+        file_pattern = rule.get('file_pattern', '*.py')
+        compiled_patterns = rule['_compiled_patterns']
+
+        for py_file in unpacked_dir.rglob(file_pattern):
+            if not py_file.is_file():
+                continue
+
+            try:
+                original_content = py_file.read_text(encoding='utf-8')
+            except UnicodeDecodeError:
+                continue
+
+            lines = original_content.splitlines(keepends=True)
+            kept_lines: list[str] = []
+            skip_indent: int | None = None
+
+            for line in lines:
+                stripped = line.rstrip('\n\r')
+
+                if skip_indent is not None:
+                    expanded = stripped.expandtabs()
+                    line_indent = len(expanded) - len(expanded.lstrip())
+                    if stripped == '' or line_indent > skip_indent:
+                        continue
+                    skip_indent = None
+
+                if any(pattern.search(line) for pattern in compiled_patterns):
+                    expanded = stripped.expandtabs()
+                    skip_indent = len(expanded) - len(expanded.lstrip())
+                    continue
+
+                kept_lines.append(line)
+
+            if len(kept_lines) != len(lines):
+                py_file.write_text(''.join(kept_lines), encoding='utf-8')
+                modified = True
+    return modified
+
+
+def clean_wheel(wheel_path: Path) -> bool:
+    """
+    Unpack the wheel at most once, remove excluded test files and strip
+    matching lines, then repack only if changes were made.
+    """
+    with ZipFile(wheel_path, 'r') as zf:
+        has_excluded_files = any(is_excluded_from_wheel(name) for name in zf.namelist())
+
+    line_removal_rules = _get_matching_rules(wheel_path.name)
+
+    if not has_excluded_files and not line_removal_rules:
+        return False
+
+    with TemporaryDirectory() as td:
+        td_path = Path(td)
+        unpack_wheel(wheel_path, dest_dir=td_path)
+        unpacked_dir = next(td_path.iterdir())
+
+        modified = False
+        if has_excluded_files:
+            if _remove_test_files_from_dir(unpacked_dir):
+                print(f'Tests removed from {wheel_path.name}')
+                modified = True
+        if line_removal_rules:
+            if _strip_lines_from_dir(unpacked_dir, line_removal_rules):
+                print(f'Stripped lines from {wheel_path.name}')
+                modified = True
+
+        if not modified:
+            return False
+
+        dest_dir = wheel_path.parent
+        before = {p.resolve() for p in dest_dir.glob("*.whl")}
+        pack_wheel(unpacked_dir, dest_dir=dest_dir)
+        # Repacking may restore the wheel's original (non-platform-specific) name.
+        # Move the repacked wheel back to the platform-specific path.
+        after = {p.resolve() for p in dest_dir.glob("*.whl")}
+        new_files = sorted(after - before, key=lambda p: p.stat().st_mtime, reverse=True)
+
+        if new_files:
+            shutil.move(str(new_files[0]), str(wheel_path))
+
+    return True
+
 def add_dependency(dependencies: dict[str, str], sizes: dict[str, VersionedWheelSizes], wheel: Path) -> None:
     project_metadata = extract_metadata(wheel)
     project_name = normalize_project_name(project_metadata['Name'])
@@ -368,23 +456,22 @@ def main():
     dependencies: dict[str, str] = {}
     sizes: dict[str, VersionedWheelSizes] = {}
 
+    # Handle wheels already in the built directory
+    for wheel in iter_wheels(built_wheels_dir):
+        clean_wheel(wheel)
+        add_dependency(dependencies, sizes, wheel)
+
     # Handle wheels currently in the external directory and move them to the built directory if they were modified
     for wheel in iter_wheels(external_wheels_dir):
-        was_modified = remove_test_files(wheel)
+        was_modified = clean_wheel(wheel)
         if was_modified:
-            # A modified wheel is no longer external → move it to built directory
             new_path = built_wheels_dir / wheel.name
             wheel.rename(new_path)
             wheel = new_path
             print(f'Moved {wheel.name} to built directory')
 
         add_dependency(dependencies, sizes, wheel)
 
-    # Handle wheels already in the built directory
-    for wheel in iter_wheels(built_wheels_dir):
-        remove_test_files(wheel)
-        add_dependency(dependencies, sizes, wheel)
-
     output_path = MOUNT_DIR / 'sizes.json'
     with output_path.open('w', encoding='utf-8') as fp:
         json.dump(sizes, fp, indent=2, sort_keys=True)

.builders/scripts/lines_to_remove.toml

@@ -0,0 +1,21 @@
+# Rules for stripping specific lines from wheel contents
+# Each rule specifies a package pattern and line patterns to remove
+
+# Example: add another rule like this
+# [[rules]]
+# package_pattern = "some_foo*"
+# file_pattern = "*.py"
+# line_patterns = [
+#     "^\\s*# bar:",
+#     "^\\s*baz\\(",
+# ]
+
+
+[[rules]]
+# pysnmp_mibs contains many MIB definition files with debug/documentation.
+# These lines are not need because we never set loadTexts to True.
+package_pattern = "pysnmp_mibs*"
+file_pattern = "*.py"
+line_patterns = [
+    "^\\s*if mibBuilder\\.loadTexts:",
+]

.github/chainguard/self.gitlab.release.branches.sts.yaml

@@ -0,0 +1,39 @@
+# Policy for: .gitlab-ci.yml release-auto job in DataDog/integrations-core
+#
+# This policy grants the GitLab CI release-auto job permission to push
+# release tags to this repository. The job runs on pushes to the master
+# branch and maintenance branches (e.g. 1.2.x).
+#
+# Naming convention:
+#   self:     Only this repository (DataDog/integrations-core) can use this policy
+#   gitlab:   This policy is for GitLab CI (not GitHub Actions)
+#   release:  Grants permission to push release tags
+#   branches: Restricted to protected release branches (master and X.Y.x)
+#
+# Security model:
+#   - Pipeline must run from master or a maintenance branch (X.Y.x)
+#   - Pipeline must be triggered by a push event (not scheduled or manual)
+#   - CI config must be committed to the same branch
+#
+# Permissions granted:
+#   - contents: write  - Push release tags to the repository
+#
+# Usage in .gitlab-ci.yml:
+#   id_tokens:
+#     DDOCTOSTS_ID_TOKEN:
+#       aud: dd-octo-sts
+#   script:
+#     - GITHUB_TOKEN=$(dd-octo-sts token --scope DataDog/integrations-core --policy self.gitlab.release.branches)
+
+issuer: https://gitlab.ddbuild.io
+
+subject_pattern: project_path:DataDog/integrations-core:ref_type:branch:ref:(master|\d+\.\d+\.x)
+
+claim_pattern:
+  ci_config_ref_uri: gitlab\.ddbuild\.io/DataDog/integrations-core//\.gitlab-ci\.yml@refs/heads/(master|\d+\.\d+\.x)
+  pipeline_source: push
+  project_path: DataDog/integrations-core
+  ref_path: refs/heads/(master|\d+\.\d+\.x)
+
+permissions:
+  contents: write

.github/workflows/update-build-agent-yaml.yml

@@ -0,0 +1,67 @@
+name: Update build_agent.yaml for release branches
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'Release branch to update (e.g. 7.56.x)'
+        required: true
+        type: string
+
+jobs:
+  update:
+    name: Update build_agent.yaml (${{ inputs.branch }})
+    runs-on: ubuntu-latest
+    env:
+      BRANCH: ${{ inputs.branch }}
+    steps:
+      - name: Checkout release branch
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ inputs.branch }}
+
+      - name: Verify preconditions
+        run: |-
+          if ! grep -qE '^\s+branch:\s+main\s*$' .gitlab/build_agent.yaml; then
+            echo "build_agent.yaml already points to the release branch — nothing to do."
+            exit 0
+          fi
+          if ! git ls-remote --exit-code --heads https://github.com/DataDog/datadog-agent.git "$BRANCH" >/dev/null 2>&1; then
+            echo "::error::Agent branch '$BRANCH' does not exist in DataDog/datadog-agent yet."
+            exit 1
+          fi
+          echo "needs_update=true" >> "$GITHUB_ENV"
+
+      - name: Update build_agent.yaml
+        if: env.needs_update == 'true'
+        run: |-
+          sed -i "s/^    branch: main$/    branch: $BRANCH/" .gitlab/build_agent.yaml
+
+      - name: Create token
+        if: env.needs_update == 'true'
+        uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1
+        id: token-generator
+        with:
+          app-id: ${{ secrets.DD_AGENT_INTEGRATIONS_BOT_APP_ID }}
+          private-key: ${{ secrets.DD_AGENT_INTEGRATIONS_BOT_PRIVATE_KEY }}
+          repositories: integrations-core
+
+      - name: Create Pull Request
+        if: env.needs_update == 'true'
+        uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
+        with:
+          token: ${{ steps.token-generator.outputs.token }}
+          title: "Update build_agent.yaml to use agent branch: ${{ inputs.branch }}"
+          commit-message: "Update build_agent.yaml to use agent branch: ${{ inputs.branch }}"
+          branch: bot/update-build-agent-yaml-${{ inputs.branch }}
+          delete-branch: true
+          base: ${{ inputs.branch }}
+          labels: bot,qa/skip-qa
+          draft: false
+          body: |
+            ### What does this PR do?
+            Updates `.gitlab/build_agent.yaml` to point to the `${{ inputs.branch }}` agent branch
+            instead of `main`, so release builds trigger the correct downstream pipeline.
+
+            This PR was automatically generated by the following workflow:
+            ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

.gitlab-ci.yml

@@ -41,6 +41,9 @@ notify-failed-pipeline:
 release-auto:
   stage: release
   image: $TAGGER_IMAGE
+  id_tokens:
+    DDOCTOSTS_ID_TOKEN:
+      aud: dd-octo-sts
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
       when: never