add yotta

.buildkite/generate_pipeline.py

@@ -146,6 +146,7 @@ def _parse_args(args: Optional[str] = None):
     parser.add_argument('--grpc', action="store_true")
     parser.add_argument('--env-file')
     parser.add_argument('--plugin-yaml')
+    parser.add_argument('--submodule-base-branch')
     parser.add_argument('--dependency', nargs='?', const='', default='all')
 
     parsed_args, _ = parser.parse_known_args(args_list)
@@ -190,6 +191,11 @@ def _parse_args(args: Optional[str] = None):
         extra_args.append('--grpc')
     if parsed_args.env_file:
         extra_args.append(f'--env-file {parsed_args.env_file}')
+    if parsed_args.plugin_yaml:
+        extra_args.append(f'--plugin-yaml {parsed_args.plugin_yaml}')
+    if parsed_args.submodule_base_branch:
+        extra_args.append(
+            f'--submodule-base-branch {parsed_args.submodule_base_branch}')
     if parsed_args.dependency != 'all':
         space = ' ' if parsed_args.dependency else ''
         extra_args.append(f'--dependency{space}{parsed_args.dependency}')
@@ -198,8 +204,9 @@ def _parse_args(args: Optional[str] = None):
 
 
 def _extract_marked_tests(
-        file_path: str, args: str
-) -> Dict[str, Tuple[List[str], List[str], List[Optional[str]]]]:
+    file_path: str, args: str
+) -> Dict[str, Tuple[List[str], List[str], List[Optional[str]], List[str],
+                     List[bool]]]:
     """Extract test functions and filter clouds using pytest.mark
     from a Python test file.
 
@@ -212,6 +219,10 @@ def _extract_marked_tests(
     and run for hours. This makes it hard to visualize the test results and
     rerun failures. Additionally, the parallelism would be controlled by pytest
     instead of the buildkite job queue.
+
+    Returns:
+        Dict mapping function_name to tuple of:
+        (clouds, queues, params, extra_args, no_auto_retry_flags)
     """
     # Args are already in the format pytest expects (cloud names like --lambda)
     cmd = f'pytest {file_path} --collect-only {args}'
@@ -259,6 +270,7 @@ def _extract_marked_tests(
         run_on_cloud_kube_backend = ('resource_heavy' in marks and
                                      'kubernetes' in default_clouds_to_run)
         benchmark_test = 'benchmark' in marks
+        no_auto_retry = 'no_auto_retry' in marks
 
         for mark in marks:
             if mark not in PYTEST_TO_CLOUD_KEYWORD:
@@ -302,20 +314,19 @@ def _extract_marked_tests(
             for cloud in final_clouds_to_include
         ], param_list, [
             extra_args for _ in range(len(final_clouds_to_include))
-        ])
+        ], [no_auto_retry for _ in range(len(final_clouds_to_include))])
 
     return function_cloud_map
 
 
-def _generate_pipeline(test_file: str,
-                       args: str,
-                       auto_retry: bool = False) -> Dict[str, Any]:
+def _generate_pipeline(test_file: str, args: str) -> Dict[str, Any]:
     """Generate a Buildkite pipeline from test files."""
     steps = []
     generated_steps_set = set()
     function_cloud_map = _extract_marked_tests(test_file, args)
     for test_function, clouds_queues_param in function_cloud_map.items():
-        for cloud, queue, param, extra_args in zip(*clouds_queues_param):
+        for cloud, queue, param, extra_args, no_auto_retry in zip(
+                *clouds_queues_param):
             label = f'{test_function} on {cloud}'
             command = f'pytest {test_file}::{test_function} --{cloud}'
             if param:
@@ -328,6 +339,7 @@ def _generate_pipeline(test_file: str,
                 continue
             if 'PYTHON_VERSION' in os.environ:
                 command = f'PYTHONPATH="$PWD:$PYTHONPATH" {command}'
+
             step = {
                 'label': label,
                 'command': command,
@@ -338,7 +350,15 @@ def _generate_pipeline(test_file: str,
                     'queue': queue
                 }
             }
-            if auto_retry:
+            if no_auto_retry:
+                # Disable automatic retries but allow manual retries.
+                step['retry'] = {
+                    'automatic': False,
+                    'manual': {
+                        'allowed': True
+                    }
+                }
+            else:
                 step['retry'] = {
                     # Automatically retry 2 times on any failure by default.
                     'automatic': True
@@ -391,7 +411,7 @@ def _convert_release(test_files: List[str], args: str, trigger_command: str):
     output_file_pipelines = []
     for test_file in test_files:
         print(f'Converting {test_file} to {yaml_file_path}')
-        pipeline = _generate_pipeline(test_file, args, auto_retry=True)
+        pipeline = _generate_pipeline(test_file, args)
         output_file_pipelines.append(pipeline)
         print(f'Converted {test_file} to {yaml_file_path}\n\n')
     # Enable all clouds by default for release pipeline.
@@ -462,11 +482,10 @@ def _convert_quick_tests_core(test_files: List[str], args: str,
                         branch != 'master'):
                     continue
                 pipeline = _generate_pipeline(test_file,
-                                              args + f' --base-branch {branch}',
-                                              auto_retry=True)
+                                              args + f' --base-branch {branch}')
                 output_file_pipelines.append(pipeline)
         else:
-            pipeline = _generate_pipeline(test_file, args, auto_retry=True)
+            pipeline = _generate_pipeline(test_file, args)
             output_file_pipelines.append(pipeline)
         print(f'Converted {test_file} to {yaml_file_path}\n\n')
     _dump_pipeline_to_file(yaml_file_path,

.buildkite/test_buildkite_pipeline_generation.py

@@ -128,6 +128,61 @@ def _extract_test_names_from_pipeline(pipeline_path):
     return test_names
 
 
+def _extract_steps_from_pipeline(pipeline_path):
+    """Extract all steps from a pipeline YAML file."""
+    with open(pipeline_path, 'r') as f:
+        pipeline = yaml.safe_load(f)
+
+    all_steps = []
+    for group in pipeline['steps']:
+        if 'steps' in group:
+            all_steps.extend(group['steps'])
+        else:
+            all_steps.append(group)
+    return all_steps
+
+
+def test_no_auto_retry_marker():
+    """Test that no_auto_retry marker works correctly.
+
+    This test uses the actual test_kubernetes_container_status_unknown_status_refresh
+    test which has the marker applied.
+    """
+    # Generate pipeline for the specific test
+    env = dict(os.environ)
+    env['PYTHONPATH'] = f"{pathlib.Path.cwd()}/tests:{env.get('PYTHONPATH', '')}"
+
+    subprocess.run([
+        'python', '.buildkite/generate_pipeline.py', '--args', '--kubernetes',
+        '--file_pattern', 'test_cluster_job'
+    ],
+                   env=env,
+                   check=True)
+
+    # Check the generated pipeline
+    pipeline_path = pathlib.Path('.buildkite/pipeline_smoke_tests_release.yaml')
+    steps = _extract_steps_from_pipeline(pipeline_path)
+
+    # Find steps for test_kubernetes_container_status_unknown_status_refresh
+    target_steps = [
+        s for s in steps
+        if 'test_kubernetes_container_status_unknown_status_refresh' in s.get(
+            'label', '')
+    ]
+
+    # Should have exactly 1 step
+    assert len(target_steps) == 1, \
+        f"Expected 1 step, got {len(target_steps)}"
+
+    # Verify no_auto_retry is applied
+    step = target_steps[0]
+    retry = step.get('retry', {})
+    assert retry.get('automatic') is False, \
+        f"no_auto_retry step should have automatic=False: {retry}"
+    assert retry.get('manual', {}).get('allowed') is True, \
+        f"no_auto_retry step should allow manual retry: {retry}"
+
+
 @pytest.mark.parametrize('args', [
     '',
     '--aws',

.cursor/worktrees.json

@@ -0,0 +1,8 @@
+{
+  "setup-worktree": [
+    "uv venv --seed --python 3.11",
+    "uv pip install -e \".[all]\" --prerelease=allow",
+    "uv pip install -r requirements-dev.txt",
+    "npm --prefix sky/dashboard install && npm --prefix sky/dashboard run build"
+  ]
+}

.github/workflows/compile-protos-check.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8"]
+        python-version: ["3.9"]
     steps:
     - uses: actions/checkout@v3
     - name: Install the latest version of uv

.github/workflows/format.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8"]
+        python-version: ["3.9"]
     steps:
     - uses: actions/checkout@v3
     - name: Install the latest version of uv

.github/workflows/mypy.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8"]
+        python-version: ["3.9"]
     steps:
     - uses: actions/checkout@v3
     - name: Install the latest version of uv