diff --git a/aws/lambda/pytorch-auto-revert/Makefile b/aws/lambda/pytorch-auto-revert/Makefile index da7e270a2b..556b3c99dd 100644 --- a/aws/lambda/pytorch-auto-revert/Makefile +++ b/aws/lambda/pytorch-auto-revert/Makefile @@ -23,7 +23,7 @@ run-local: venv/bin/python .PHONY: run-local-workflows run-local-workflows: venv/bin/python - venv/bin/python -m pytorch_auto_revert workflows pull.yml + venv/bin/python -m pytorch_auto_revert autorevert-checker Lint trunk pull inductor linux-binary-manywheel --hours 4320 --ignore-common-errors deployment.zip: mkdir -p deployment diff --git a/aws/lambda/pytorch-auto-revert/pytorch_auto_revert/__main__.py b/aws/lambda/pytorch-auto-revert/pytorch_auto_revert/__main__.py index 4433aba442..8ab2ca6954 100644 --- a/aws/lambda/pytorch-auto-revert/pytorch_auto_revert/__main__.py +++ b/aws/lambda/pytorch-auto-revert/pytorch_auto_revert/__main__.py @@ -95,6 +95,11 @@ def get_opts() -> argparse.Namespace: action="store_true", help="Show what would be restarted without actually doing it (use with --do-restart)", ) + workflow_parser.add_argument( + "--ignore-common-errors", + action="store_true", + help="Ignore common errors in autorevert patterns (e.g., 'No tests found')", + ) # workflow-restart-checker subcommand workflow_restart_parser = subparsers.add_parser( @@ -159,8 +164,21 @@ def main(*args, **kwargs) -> None: "ClickHouse connection test failed. Please check your configuration." ) - if opts.subcommand == "lambda": - print("TODO: run lambda flow") + if opts.subcommand is None: + autorevert_checker( + [ + "Lint", + "trunk", + "pull", + "inductor", + "linux-binary-manywheel", + ], + hours=2, + verbose=True, + do_restart=True, + dry_run=False, + ignore_common_errors=True, + ) elif opts.subcommand == "autorevert-checker": autorevert_checker( opts.workflows, @@ -168,6 +186,7 @@ def main(*args, **kwargs) -> None: verbose=opts.verbose, do_restart=opts.do_restart, dry_run=opts.dry_run, + ignore_common_errors=opts.ignore_common_errors, ) elif opts.subcommand == "workflow-restart-checker": workflow_restart_checker(opts.workflow, commit=opts.commit, days=opts.days) diff --git a/aws/lambda/pytorch-auto-revert/pytorch_auto_revert/autorevert_checker.py b/aws/lambda/pytorch-auto-revert/pytorch_auto_revert/autorevert_checker.py index cc31638ef4..2793fa39ca 100644 --- a/aws/lambda/pytorch-auto-revert/pytorch_auto_revert/autorevert_checker.py +++ b/aws/lambda/pytorch-auto-revert/pytorch_auto_revert/autorevert_checker.py @@ -63,14 +63,20 @@ def get_job_base_names(self) -> Set[str]: class AutorevertPatternChecker: """Detects autorevert patterns in workflow job failures.""" - def __init__(self, workflow_names: List[str] = None, lookback_hours: int = 48): + def __init__( + self, + workflow_names: List[str] = None, + lookback_hours: int = 48, + ignore_classification_rules: Set[str] = None, + ): self.workflow_names = workflow_names or [] self.lookback_hours = lookback_hours self._workflow_commits_cache: Dict[str, List[CommitJobs]] = {} self._commit_history = None + self._ignore_classification_rules = ignore_classification_rules or set() def get_workflow_commits(self, workflow_name: str) -> List[CommitJobs]: - """Get workflow commits for a specific workflow, fetching if needed.""" + """Get workflow commits for a specific workflow, fetching if needed. From newer to older""" if workflow_name not in self._workflow_commits_cache: self._fetch_workflow_data() return self._workflow_commits_cache.get(workflow_name, []) @@ -90,7 +96,7 @@ def commit_history(self) -> List[Dict]: return self._commit_history or [] def _fetch_workflow_data(self): - """Fetch workflow job data from ClickHouse for all workflows in batch.""" + """Fetch workflow job data from ClickHouse for all workflows in batch. From newer to older""" if not self.workflow_names: return @@ -203,25 +209,25 @@ def _find_last_commit_with_job( self, commits: Iterable[CommitJobs], job_name: str ) -> Optional[Tuple[CommitJobs, List[JobResult]]]: """ - Find the last commit in the iterable that has a job with the specified name. + Find the first commit (in the provided iteration order) that has a job with the specified name. Args: commits: Iterable of CommitJobs to search job_name: The job name to look for Returns: - The last CommitJobs object that contains the specified job, or None if not found + The first CommitJobs object (per the iterable's order) that contains the specified job, or None if not found """ job_results = [] for commit in commits: for job in commit.jobs: if job.name.split("(")[0] == job_name: # Normalize job name job_results.append(job) - if job_results: - return ( - commit, - job_results, - ) + if job_results: + return ( + commit, + job_results, + ) return None, None def detect_autorevert_pattern_workflow(self, workflow_name: str) -> List[Dict]: @@ -266,6 +272,10 @@ def detect_autorevert_pattern_workflow(self, workflow_name: str) -> List[Dict]: suspected_failure_class_rule, suspected_failure_job_name, ) in suspected_failures: + if suspected_failure_class_rule in self._ignore_classification_rules: + # Skip ignored classification rules + continue + newer_commit_same_job, newer_same_jobs = ( self._find_last_commit_with_job( (commits[j] for j in range(i - 1, -1, -1)), @@ -304,18 +314,8 @@ def detect_autorevert_pattern_workflow(self, workflow_name: str) -> List[Dict]: # No older commit with the same job found continue - if any( - j.name.split("(")[0] != job_name - for j in last_commit_with_same_job.failed_jobs - ): - # newr commit has the same job failing - continue - - if any( - j.classification_rule == suspected_failure_class_rule - for j in last_same_jobs - ): - # The last commit with the same job has the same failure classification + if any(j.classification_rule == failure_rule for j in last_same_jobs): + # The older commit has the same job failing with same rule continue patterns.append( diff --git a/aws/lambda/pytorch-auto-revert/pytorch_auto_revert/testers/autorevert.py b/aws/lambda/pytorch-auto-revert/pytorch_auto_revert/testers/autorevert.py index bbba95a627..87a0a90d24 100644 --- a/aws/lambda/pytorch-auto-revert/pytorch_auto_revert/testers/autorevert.py +++ b/aws/lambda/pytorch-auto-revert/pytorch_auto_revert/testers/autorevert.py @@ -10,9 +10,20 @@ def autorevert_checker( verbose: bool = False, do_restart: bool = False, dry_run: bool = False, + ignore_common_errors=True, ): + common_errors = { + "GHA error", + "GHA timeout", + "sccache error", + } + # Initialize checker - checker = AutorevertPatternChecker(workflow_names, hours) + checker = AutorevertPatternChecker( + workflow_names, + hours, + ignore_classification_rules=common_errors if ignore_common_errors else set(), + ) # Fetch data if verbose: @@ -108,7 +119,7 @@ def autorevert_checker( "category", "uncategorized" ) print( - f"✓ REVERTED ({category}): {second_commit[:8]} was reverted by {revert_result['revert_sha'][:8]} " + f"✓ REVERTED ({category}): {second_commit} was reverted by {revert_result['revert_sha'][:8]} " f"after {revert_result['hours_after_target']:.1f} hours" ) reverted_patterns.append(pattern)