[autorevert] Add workflow restarts (#6962)

###  Summary

Adds workflow restart capability to the PyTorch auto-revert tool,
enabling automatic re-running of workflows for commits that match
autorevert patterns but haven't been reverted yet.

###  Changes

  - Added restart methods to WorkflowRestartChecker:
- restart_workflow(): Restarts a workflow for a specific commit with
duplicate prevention
    - Checks ClickHouse for existing restarts before attempting

  - Enhanced autorevert-checker command:
    - Added --do-restart flag to enable automatic workflow restarts
    - Added --dry-run flag to preview restart actions without execution
- Restarts workflows only for non-reverted commits matching autorevert
patterns

  - Fixed workflow naming consistency:
- Normalized workflow names by removing .yml extension for ClickHouse
queries
    - Added .yml extension only for GitHub API calls

  - Updated do-restart command:
    - Now requires commit SHA (removed unused restart_latest_workflow)
    - Leverages same restart logic with duplicate prevention

###  Usage

```
  # Check patterns and restart workflows
  python -m pytorch_auto_revert autorevert-checker pull trunk --do-restart

  # Dry run to preview restarts
  python -m pytorch_auto_revert autorevert-checker pull trunk --do-restart --dry-run

  # Manual restart
  python -m pytorch_auto_revert do-restart trunk abc123def
  ```
  
  ### Testing
  
  ```
  python -m pytorch_auto_revert autorevert-checker inductor --hours 12 --do-restart --dry-run
Fetching workflow data for 1 workflows since 2025-07-30T22:47:17.357776...
Found 19 commits with job data for workflow 'inductor'
✓ 1 AUTOREVERT PATTERN DETECTED

Pattern #1:
Failure rule: 'GHA error'
Recent commits with failure: f89c28cc 5b2ad927
Older commit without failure: 7a4167a1
✗ NOT REVERTED: 5b2ad9279cb2e440d45253d28f2101a75fd42344 was not reverted
  ⟳ ALREADY RESTARTED: inductor for 5b2ad927

==================================================
SUMMARY STATISTICS
==================================================
Workflow(s): inductor
Timeframe: 12 hours
Commits checked: 19
Auto revert patterns detected: 1
Actual reverts inside auto revert patterns detected (precision): 0 (0.0%)
Total revert commits in period: 0

Total reverts excluding ghfirst: 0
No non-ghfirst reverts found in the period
Per workflow precision:
  inductor: 0 reverts out of 1 patterns (0.0%) [excluding ghfirst: 0 (0.0%)]
  ```

Author: izaitsevfb

aws/lambda/pytorch-auto-revert/pytorch_auto_revert/__main__.py

@@ -84,6 +84,16 @@ def get_opts() -> argparse.Namespace:
         action="store_true",
         help="Show detailed output including commit summaries",
     )
+    workflow_parser.add_argument(
+        "--do-restart",
+        action="store_true",
+        help="Actually restart workflows for detected autorevert patterns",
+    )
+    workflow_parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Show what would be restarted without actually doing it (use with --do-restart)",
+    )
 
     # workflow-restart-checker subcommand
     workflow_restart_parser = subparsers.add_parser(
@@ -146,7 +156,13 @@ def main(*args, **kwargs) -> None:
     if opts.subcommand == "lambda":
         print("TODO: run lambda flow")
     elif opts.subcommand == "autorevert-checker":
-        autorevert_checker(opts.workflows, hours=opts.hours, verbose=opts.verbose)
+        autorevert_checker(
+            opts.workflows,
+            hours=opts.hours,
+            verbose=opts.verbose,
+            do_restart=opts.do_restart,
+            dry_run=opts.dry_run,
+        )
     elif opts.subcommand == "workflow-restart-checker":
         workflow_restart_checker(opts.workflow, commit=opts.commit, days=opts.days)
     elif opts.subcommand == "do-restart":

aws/lambda/pytorch-auto-revert/pytorch_auto_revert/testers/autorevert.py

@@ -1,10 +1,15 @@
 from collections import defaultdict
 
 from ..autorevert_checker import AutorevertPatternChecker
+from ..workflow_checker import WorkflowRestartChecker
 
 
 def autorevert_checker(
-    workflow_names: list[str], hours: int = 48, verbose: bool = False
+    workflow_names: list[str],
+    hours: int = 48,
+    verbose: bool = False,
+    do_restart: bool = False,
+    dry_run: bool = False,
 ):
     # Initialize checker
     checker = AutorevertPatternChecker(workflow_names, hours)
@@ -67,6 +72,10 @@ def autorevert_checker(
             workflow_names=[], lookback_hours=hours * 2
         )
 
+        # Initialize workflow restart checker if needed
+        restart_checker = WorkflowRestartChecker() if do_restart else None
+        restarted_commits = []
+
         # Track reverts
         reverted_patterns = []
 
@@ -106,6 +115,37 @@ def autorevert_checker(
             else:
                 print(f"✗ NOT REVERTED: {second_commit} was not reverted")
 
+                # Try to restart workflow if --do-restart flag is set and not already reverted
+                if do_restart and restart_checker:
+                    # Restart for the second commit (older of the two failures)
+                    workflow_name = pattern["workflow_name"]
+
+                    # Check if already restarted
+                    if restart_checker.has_restarted_workflow(
+                        workflow_name, second_commit
+                    ):
+                        print(
+                            f"  ⟳ ALREADY RESTARTED: {workflow_name} for {second_commit[:8]}"
+                        )
+                    elif dry_run:
+                        print(
+                            f"  ⟳ DRY RUN: Would restart {workflow_name} for {second_commit[:8]}"
+                        )
+                        restarted_commits.append((workflow_name, second_commit))
+                    else:
+                        success = restart_checker.restart_workflow(
+                            workflow_name, second_commit
+                        )
+                        if success:
+                            print(
+                                f"  ✓ RESTARTED: {workflow_name} for {second_commit[:8]}"
+                            )
+                            restarted_commits.append((workflow_name, second_commit))
+                        else:
+                            print(
+                                f"  ✗ FAILED TO RESTART: {workflow_name} for {second_commit[:8]}"
+                            )
+
             if verbose:
                 print(f"Failed jobs ({len(pattern['failed_job_names'])}):")
                 for job in pattern["failed_job_names"][:5]:
@@ -216,6 +256,12 @@ def autorevert_checker(
                     f"  - {pattern['failure_rule']}: {second_commit[:8]} ({category})"
                 )
 
+        # Show restart summary if applicable
+        if do_restart and restarted_commits:
+            print(f"\nRestarted workflows: {len(restarted_commits)}")
+            for workflow, commit in restarted_commits:
+                print(f"  - {workflow} for {commit[:8]}")
+
     else:
         print("✗ No autorevert patterns detected")
 

aws/lambda/pytorch-auto-revert/pytorch_auto_revert/testers/do_restart.py

@@ -1,14 +1,9 @@
 from ..workflow_checker import WorkflowRestartChecker
 
 
-def do_restart_workflow(workflow: str, commit: str = None) -> None:
+def do_restart_workflow(workflow: str, commit: str) -> None:
     checker = WorkflowRestartChecker()
 
-    if commit:
-        # Restart specific commit
-        success = checker.restart_workflow(workflow, commit)
-        print(f"Commit {commit}: {'✓ RESTARTED' if success else '✗ Not restarted'}")
-    else:
-        # Restart latest commit
-        success = checker.restart_latest_workflow(workflow)
-        print(f"Latest commit: {'✓ RESTARTED' if success else '✗ Not restarted'}")
+    # Restart specific commit
+    success = checker.restart_workflow(workflow, commit)
+    print(f"Commit {commit}: {'✓ RESTARTED' if success else '✗ Not restarted'}")

aws/lambda/pytorch-auto-revert/pytorch_auto_revert/workflow_checker.py

@@ -2,6 +2,7 @@
 WorkflowRestartChecker for querying restarted workflows via ClickHouse.
 """
 
+import logging
 from datetime import datetime, timedelta
 from typing import Dict, Set
 
@@ -11,21 +12,25 @@
 class WorkflowRestartChecker:
     """Check if workflows have been restarted using ClickHouse."""
 
-    def __init__(self):
+    def __init__(self, repo_owner: str = "pytorch", repo_name: str = "pytorch"):
         self._cache: Dict[str, bool] = {}
+        self.repo_owner = repo_owner
+        self.repo_name = repo_name
 
     def has_restarted_workflow(self, workflow_name: str, commit_sha: str) -> bool:
         """
         Check if a workflow has been restarted for given commit.
 
         Args:
-            workflow_name: Name of workflow (e.g., "trunk")
+            workflow_name: Name of workflow (e.g., "trunk" or "trunk.yml")
             commit_sha: Commit SHA to check
 
         Returns:
             bool: True if workflow was restarted (workflow_dispatch with trunk/* branch)
         """
-        cache_key = f"{workflow_name}:{commit_sha}"
+        # Normalize workflow name - remove .yml extension for consistency
+        normalized_workflow_name = workflow_name.replace(".yml", "")
+        cache_key = f"{normalized_workflow_name}:{commit_sha}"
         if cache_key in self._cache:
             return self._cache[cache_key]
 
@@ -49,7 +54,7 @@ def has_restarted_workflow(self, workflow_name: str, commit_sha: str) -> bool:
                 "commit_sha": commit_sha,
                 "workflow_event": "workflow_dispatch",
                 "head_branch": f"trunk/{commit_sha}",
-                "workflow_name": workflow_name,
+                "workflow_name": normalized_workflow_name,
             },
         )
 
@@ -62,12 +67,14 @@ def get_restarted_commits(self, workflow_name: str, days_back: int = 7) -> Set[s
         Get all commits with restarted workflows in date range.
 
         Args:
-            workflow_name: Name of workflow
+            workflow_name: Name of workflow (e.g., "trunk" or "trunk.yml")
             days_back: Number of days to look back
 
         Returns:
             Set of commit SHAs that have restarted workflows
         """
+        # Normalize workflow name - remove .yml extension for consistency
+        normalized_workflow_name = workflow_name.replace(".yml", "")
         since_date = datetime.now() - timedelta(days=days_back)
 
         query = """
@@ -80,18 +87,90 @@ def get_restarted_commits(self, workflow_name: str, days_back: int = 7) -> Set[s
         """
 
         result = CHCliFactory().client.query(
-            query, {"workflow_name": workflow_name, "since_date": since_date}
+            query, {"workflow_name": normalized_workflow_name, "since_date": since_date}
         )
 
         commits = {row[0] for row in result.result_rows}
 
         # Update cache
         for commit_sha in commits:
-            cache_key = f"{workflow_name}:{commit_sha}"
+            cache_key = f"{normalized_workflow_name}:{commit_sha}"
             self._cache[cache_key] = True
 
         return commits
 
     def clear_cache(self):
         """Clear the results cache."""
         self._cache.clear()
+
+    def restart_workflow(self, workflow_name: str, commit_sha: str) -> bool:
+        """
+        Restart a workflow for a specific commit SHA.
+
+        Args:
+            workflow_name: Name of the workflow (e.g., "trunk" or "trunk.yml")
+            commit_sha: The commit SHA to restart workflow for
+
+        Returns:
+            bool: True if workflow was successfully dispatched, False otherwise
+        """
+        # Normalize workflow name
+        normalized_workflow_name = workflow_name.replace(".yml", "")
+
+        # Check if already restarted
+        if self.has_restarted_workflow(normalized_workflow_name, commit_sha):
+            logging.warning(
+                f"Workflow {normalized_workflow_name} already restarted for commit {commit_sha}"
+            )
+            return False
+
+        # Get GitHub client
+        try:
+            from .github_client_helper import GHClientFactory
+
+            if not (
+                GHClientFactory().token_auth_provided
+                or GHClientFactory().key_auth_provided
+            ):
+                logging.error("GitHub authentication not configured")
+                return False
+
+            client = GHClientFactory().client
+        except Exception as e:
+            logging.error(f"Failed to get GitHub client: {e}")
+            return False
+
+        try:
+            # Use trunk/{sha} tag format
+            tag_ref = f"trunk/{commit_sha}"
+
+            # Add .yml extension for workflow name
+            workflow_file_name = f"{normalized_workflow_name}.yml"
+
+            # Get repo and workflow objects
+            repo = client.get_repo(f"{self.repo_owner}/{self.repo_name}")
+            workflow = repo.get_workflow(workflow_file_name)
+
+            # Dispatch the workflow
+            workflow.create_dispatch(ref=tag_ref, inputs={})
+
+            # Construct the workflow runs URL
+            workflow_url = (
+                f"https://github.com/{self.repo_owner}/{self.repo_name}"
+                f"/actions/workflows/{workflow_file_name}"
+                f"?query=branch%3Atrunk%2F{commit_sha}"
+            )
+            logging.info(
+                f"Successfully dispatched workflow {normalized_workflow_name} for commit {commit_sha}\n"
+                f"  View at: {workflow_url}"
+            )
+
+            # Invalidate cache for this workflow/commit
+            cache_key = f"{normalized_workflow_name}:{commit_sha}"
+            if cache_key in self._cache:
+                del self._cache[cache_key]
+            return True
+
+        except Exception as e:
+            logging.error(f"Error dispatching workflow {normalized_workflow_name}: {e}")
+            return False

aws/lambda/pytorch-auto-revert/requirements.txt

@@ -1,3 +1,4 @@
 clickhouse-connect==0.8.14
 PyGithub==2.6.1
 python-dotenv>=1.0.0
+requests>=2.31.0