Add dependency updates report to the dep updates command and improve dep update workflow (DataDog#22855)

* Add dependency updates report to the dep updates command and improve dep update workflow

* Add changelog

* Update changelog since I used a previous PR number to generate it

* Fix formatting

* Updated the behavior of the report to get the report printed on screen if passed without a path

Author: AAraKKe

.github/workflows/update-dependencies.yml

@@ -37,7 +37,13 @@ jobs:
         repositories: integrations-core
     - name: Update dependencies
       run: |-
-        ddev dep updates --sync
+        ddev dep updates --sync --report /tmp/dep-report.md --report-type markdown
+    - name: Read dependency report
+      id: dep-summary
+      run: |-
+        echo "summary<<EOF" >> "$GITHUB_OUTPUT"
+        cat /tmp/dep-report.md >> "$GITHUB_OUTPUT"
+        echo "EOF" >> "$GITHUB_OUTPUT"
     - name: Update licenses
       run: |-
         ddev validate licenses --sync
@@ -62,11 +68,11 @@ jobs:
           ### What does this PR do?
           Update the dependencies
 
+          ${{ steps.dep-summary.outputs.summary }}
+
           This PR was automatically generated by the following workflow:
           ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
 
-          **THE CHANGELOG FILES OFTEN HAVE THE WRONG PR NUMBER. MAKE SURE TO CHECK THEM!**
-
           ### Review checklist (to be filled by reviewers)
 
           - [ ] Feature or bugfix MUST have appropriate tests (unit, integration, e2e)
@@ -79,3 +85,30 @@ jobs:
         base: master
         labels: bot,qa/skip-qa
         draft: false
+    - name: Fix changelog PR numbers
+      if: steps.cpr.outputs.pull-request-number != ''
+      env:
+        GITHUB_TOKEN: ${{ steps.token-generator.outputs.token }}
+      run: |-
+        actual_pr="${{ steps.cpr.outputs.pull-request-number }}"
+        branch="${{ steps.cpr.outputs.pull-request-branch }}"
+
+        # Resolve the actual base branch from the PR (avoids hardcoding 'master')
+        base_branch=$(gh pr view "$actual_pr" --json baseRefName -q '.baseRefName')
+
+        git fetch origin "$branch" "$base_branch"
+        git checkout "$branch"
+
+        git diff "origin/$base_branch"...HEAD --name-only --diff-filter=A | grep 'changelog\.d/' | while read -r file; do
+          ext="${file##*.}"
+          dir=$(dirname "$file")
+          git mv "$file" "$dir/$actual_pr.$ext" 2>/dev/null || true
+        done
+
+        git diff --cached --quiet || (
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git config user.name "github-actions[bot]"
+          git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }}.git"
+          git commit -m "Fix changelog PR numbers"
+          git push origin "$branch"
+        )

ddev/changelog.d/22855.added

@@ -0,0 +1,3 @@
+Added `--report` and `--report-type` options to `ddev dep updates`:
+  - Pass `--report PATH` to write a dependency update report to a file (JSON or Markdown via `--report-type`).
+  - Pass `--report` without a path to print the report as a Rich table directly in the terminal.

ddev/src/ddev/cli/dep/updates.py

@@ -16,15 +16,31 @@
     scrape_version_data,
     update_agent_dependencies,
 )
+from ddev.utils.fs import Path
 
 
 @click.command(short_help='Automatically check for dependency updates')
 @click.option('--sync', '-s', 'sync_dependencies', is_flag=True, help='Update the dependency definitions')
 @click.option('--include-security-deps', '-i', is_flag=True, help="Attempt to update security dependencies")
 @click.option('--batch-size', '-b', type=int, help='The maximum number of dependencies to upgrade if syncing')
+@click.option(
+    '--report',
+    is_flag=False,
+    flag_value='',
+    default=None,
+    metavar='PATH',
+    help='Write a dependency update report to PATH, or omit PATH to print to screen.',
+)
+@click.option(
+    '--report-type',
+    type=click.Choice(['json', 'markdown']),
+    default='json',
+    show_default=True,
+    help='Format of the report file (only applies when a PATH is given).',
+)
 @click.pass_context
 @click.pass_obj
-def updates(app, ctx, sync_dependencies, include_security_deps, batch_size):
+def updates(app, ctx, sync_dependencies, include_security_deps, batch_size, report, report_type):
     ignore_deps = set(app.repo.config.get('/overrides/dep/updates/exclude', []))
 
     if not include_security_deps:
@@ -45,6 +61,7 @@ def updates(app, ctx, sync_dependencies, include_security_deps, batch_size):
     new_dependencies = copy.deepcopy(dependencies)
     version_updates = defaultdict(lambda: defaultdict(set))
     updated_packages = set()
+    report_entries: list[dict] = []
     for name, python_versions in sorted(new_dependencies.items()):
         if name in ignore_deps:
             continue
@@ -67,20 +84,70 @@ def updates(app, ctx, sync_dependencies, include_security_deps, batch_size):
         if dependency_definition != new_dependency_definition:
             version_updates[name][package_version].add(python_version)
             updated_packages.add(name)
+            if report is not None:
+                report_entries.append(
+                    {
+                        'package': name,
+                        'old_version': str(Requirement(dependency_definition).specifier).lstrip('='),
+                        'new_version': str(package_version),
+                    }
+                )
+
+    if report is None:
+        pass
+    elif report == '':
+        _print_table_report(app, report_entries)
+    elif report_type == 'json':
+        _write_json_report(report_entries, report)
+    else:
+        _write_markdown_report(report_entries, report)
 
     if sync_dependencies:
         if updated_packages:
             update_agent_dependencies(app.repo, new_dependencies)
             ctx.invoke(sync)
             app.display_info(f'Updated {len(updated_packages)} dependencies')
+    elif updated_packages:
+        app.display_error(f"{len(updated_packages)} dependencies are out of sync:")
+        for name, versions in version_updates.items():
+            for package_version, python_versions in versions.items():
+                app.display_error(
+                    f'{name} can be updated to version {package_version} on {" and ".join(sorted(python_versions))}'
+                )
+        app.abort()
     else:
-        if updated_packages:
-            app.display_error(f"{len(updated_packages)} dependencies are out of sync:")
-            for name, versions in version_updates.items():
-                for package_version, python_versions in versions.items():
-                    app.display_error(
-                        f'{name} can be updated to version {package_version} on {" and ".join(sorted(python_versions))}'
-                    )
-            app.abort()
-        else:
-            app.display_info('All dependencies are up to date')
+        app.display_info('All dependencies are up to date')
+
+
+def _print_table_report(app, entries: list[dict]) -> None:
+    sorted_entries = sorted(entries, key=lambda e: e['package'])
+    if not sorted_entries:
+        app.display_info('No dependency version changes detected.')
+        return
+    columns = {
+        'Package': {i: e['package'] for i, e in enumerate(sorted_entries)},
+        'Old Version': {i: e['old_version'] for i, e in enumerate(sorted_entries)},
+        'New Version': {i: e['new_version'] for i, e in enumerate(sorted_entries)},
+    }
+    app.display_table('Dependency Bumps', columns, show_lines=True)
+
+
+def _write_json_report(entries: list[dict], path: str) -> None:
+    import json
+
+    Path(path).write_text(json.dumps(sorted(entries, key=lambda e: e['package']), indent=2))
+
+
+def _write_markdown_report(entries: list[dict], path: str) -> None:
+    if sorted_entries := sorted(entries, key=lambda e: e['package']):
+        lines = [
+            '### Dependency Bumps',
+            '',
+            '| Package | Old Version | New Version |',
+            '|---------|-------------|-------------|',
+        ]
+        lines.extend(f"| {e['package']} | {e['old_version']} | {e['new_version']} |" for e in sorted_entries)
+        content = '\n'.join(lines)
+    else:
+        content = '_No dependency version changes detected._'
+    Path(path).write_text(content)

ddev/tests/cli/test_dep.py

@@ -254,6 +254,135 @@ def test_batch_size(self, ddev):
         assert result.exit_code != 0
 
 
+def test_report_json(ddev, fake_repo, mock_async_http_get_json, tmp_path):
+    create_integration(fake_repo, 'foo', ['dep-a==1.0.0', 'dep-b==3.1.4'])
+    create_integration(fake_repo, 'datadog_checks_base', ['dep-a==1.0.0', 'dep-b==3.1.4'])
+    create_integration(fake_repo, 'datadog_checks_downloader', ['dep-a==1.0.0', 'dep-b==3.1.4'])
+    (fake_repo / 'agent_requirements.in').write_text('dep-a==1.0.0\ndep-b==3.1.4')
+
+    mock_async_http_get_json(
+        'https://pypi.org/pypi/dep-a/json',
+        {
+            'info': {'name': 'dep-a'},
+            'releases': {v: [{'python_version': 'py2.py3', 'requires_python': '>=2.7'}] for v in ['1.0.0', '1.2.3']},
+        },
+    )
+    mock_async_http_get_json(
+        'https://pypi.org/pypi/dep-b/json',
+        {
+            'info': {'name': 'dep-b'},
+            'releases': {v: [{'python_version': 'py2.py3', 'requires_python': '>=2.7'}] for v in ['3.1.0', '3.1.4']},
+        },
+    )
+
+    report_path = tmp_path / 'report.json'
+    result = ddev('dep', 'updates', '--sync', '--report', str(report_path), '--report-type', 'json')
+
+    assert result.exit_code == 0
+    data = json.loads(report_path.read_text())
+    assert len(data) == 1
+    assert data[0]['package'] == 'dep-a'
+    assert data[0]['old_version'] == '1.0.0'
+    assert data[0]['new_version'] == '1.2.3'
+
+
+def test_report_markdown(ddev, fake_repo, mock_async_http_get_json, tmp_path):
+    create_integration(fake_repo, 'foo', ['dep-a==1.0.0', 'dep-b==3.1.4'])
+    create_integration(fake_repo, 'datadog_checks_base', ['dep-a==1.0.0', 'dep-b==3.1.4'])
+    create_integration(fake_repo, 'datadog_checks_downloader', ['dep-a==1.0.0', 'dep-b==3.1.4'])
+    (fake_repo / 'agent_requirements.in').write_text('dep-a==1.0.0\ndep-b==3.1.4')
+
+    mock_async_http_get_json(
+        'https://pypi.org/pypi/dep-a/json',
+        {
+            'info': {'name': 'dep-a'},
+            'releases': {v: [{'python_version': 'py2.py3', 'requires_python': '>=2.7'}] for v in ['1.0.0', '1.2.3']},
+        },
+    )
+    mock_async_http_get_json(
+        'https://pypi.org/pypi/dep-b/json',
+        {
+            'info': {'name': 'dep-b'},
+            'releases': {v: [{'python_version': 'py2.py3', 'requires_python': '>=2.7'}] for v in ['3.1.0', '3.1.4']},
+        },
+    )
+
+    report_path = tmp_path / 'report.md'
+    result = ddev('dep', 'updates', '--sync', '--report', str(report_path), '--report-type', 'markdown')
+
+    assert result.exit_code == 0
+    content = report_path.read_text()
+    assert '### Dependency Bumps' in content
+    assert '| Package | Old Version | New Version |' in content
+    assert '| dep-a | 1.0.0 | 1.2.3 |' in content
+
+
+def test_report_screen(ddev, fake_repo, mock_async_http_get_json):
+    create_integration(fake_repo, 'foo', ['dep-a==1.0.0', 'dep-b==3.1.4'])
+    create_integration(fake_repo, 'datadog_checks_base', ['dep-a==1.0.0', 'dep-b==3.1.4'])
+    create_integration(fake_repo, 'datadog_checks_downloader', ['dep-a==1.0.0', 'dep-b==3.1.4'])
+    (fake_repo / 'agent_requirements.in').write_text('dep-a==1.0.0\ndep-b==3.1.4')
+
+    mock_async_http_get_json(
+        'https://pypi.org/pypi/dep-a/json',
+        {
+            'info': {'name': 'dep-a'},
+            'releases': {v: [{'python_version': 'py2.py3', 'requires_python': '>=2.7'}] for v in ['1.0.0', '1.2.3']},
+        },
+    )
+    mock_async_http_get_json(
+        'https://pypi.org/pypi/dep-b/json',
+        {
+            'info': {'name': 'dep-b'},
+            'releases': {v: [{'python_version': 'py2.py3', 'requires_python': '>=2.7'}] for v in ['3.1.0', '3.1.4']},
+        },
+    )
+
+    result = ddev('dep', 'updates', '--sync', '--report')
+
+    assert result.exit_code == 0
+    assert 'dep-a' in result.output
+    assert '1.0.0' in result.output
+    assert '1.2.3' in result.output
+
+
+def test_report_screen_no_updates(ddev, fake_repo, mock_async_http_get_json):
+    create_integration(fake_repo, 'foo', ['dep-a==1.2.3'])
+    (fake_repo / 'agent_requirements.in').write_text('dep-a==1.2.3')
+
+    mock_async_http_get_json(
+        'https://pypi.org/pypi/dep-a/json',
+        {
+            'info': {'name': 'dep-a'},
+            'releases': {v: [{'python_version': 'py2.py3', 'requires_python': '>=2.7'}] for v in ['1.0.0', '1.2.3']},
+        },
+    )
+
+    result = ddev('dep', 'updates', '--sync', '--report')
+
+    assert result.exit_code == 0
+    assert 'No dependency version changes detected.' in result.output
+
+
+def test_report_no_updates(ddev, fake_repo, mock_async_http_get_json, tmp_path):
+    create_integration(fake_repo, 'foo', ['dep-a==1.2.3'])
+    (fake_repo / 'agent_requirements.in').write_text('dep-a==1.2.3')
+
+    mock_async_http_get_json(
+        'https://pypi.org/pypi/dep-a/json',
+        {
+            'info': {'name': 'dep-a'},
+            'releases': {v: [{'python_version': 'py2.py3', 'requires_python': '>=2.7'}] for v in ['1.0.0', '1.2.3']},
+        },
+    )
+
+    report_path = tmp_path / 'report.md'
+    result = ddev('dep', 'updates', '--sync', '--report', str(report_path), '--report-type', 'markdown')
+
+    assert result.exit_code == 0
+    assert report_path.read_text() == '_No dependency version changes detected._'
+
+
 @pytest.fixture
 def mock_async_http_get_json():
     """Mock `get` responses assuming a JSON value is returned in the body.