Merge branch 'add/juicefs-mounting' of https://github.com/transformerlab/transformerlab-app into add/juicefs-mounting

Author: deep1401

.agents/skills/transformerlab-cli/SKILL.md

@@ -162,19 +162,26 @@ github_repo_branch: main
 
 To validate without creating, use `lab task add ./my-task --dry-run`.
 
-### Editing task.yaml for an existing task
+### Editing an existing task
 
-Use `lab task edit` to update the `task.yaml` for a task that already exists on the server:
+Use `lab task edit` to update an existing task on the server. Three modes:
 
 ```bash
-# Interactive editor flow
+# Interactive editor flow (opens $EDITOR with current task.yaml)
 lab task edit TASK_ID
 
-# Apply a local task.yaml directly
+# Replace ONLY task.yaml (leaves main.py and other attachments untouched)
 lab task edit TASK_ID --from-file ./task.yaml --no-interactive
+
+# Replace task.yaml AND attachments (main.py, configs, etc.) from a directory
+lab task edit TASK_ID --from-dir ./my-task --no-interactive
 ```
 
-`lab task edit` expects a `TASK_ID` and supports `--from-file`, `--no-interactive`, and `--timeout`.
+**Choosing between `--from-file` and `--from-dir`:**
+- `--from-file <task.yaml>` only updates the YAML. Use it when you're tweaking parameters or the `run` command and the rest of the task directory is unchanged.
+- `--from-dir <directory>` zips and uploads the whole directory (must contain `task.yaml`), replacing the task's files server-side. Use it when you've also modified `main.py` or any sibling files. **If the task's `run` references a script (e.g. `python main.py`), reuploading just the YAML will leave the old script in place — use `--from-dir` to keep them in sync.**
+
+`--from-file` and `--from-dir` are mutually exclusive. `lab task edit` also supports `--no-interactive`, `--dry-run` (with `--from-dir`), and `--timeout`.
 
 ### Uploading extra files to an existing task
 
@@ -690,7 +697,7 @@ This applies to launching jobs, fetching logs, checking cluster status, and ever
 | `lab task info <id>` | Get task details | Yes |
 | `lab task init` | Scaffold `task.yaml` + `main.py` in the current directory (`--interactive` to prompt) | No |
 | `lab task add [dir]` | Add task from directory or `--from-git` URL (`--no-interactive`, `--dry-run`) | Yes |
-| `lab task edit <id>` | Edit an existing task's `task.yaml` (`--from-file`, `--no-interactive`, `--timeout`) | Yes |
+| `lab task edit <id>` | Edit an existing task. `--from-file <task.yaml>` for YAML-only; `--from-dir <dir>` to also replace attachments like `main.py` (`--no-interactive`, `--dry-run`, `--timeout`) | Yes |
 | `lab task upload <id> <path>` | Upload files/directories into an existing task (`--no-interactive`) | Yes |
 | `lab task delete <id>` | Delete a task (`--no-interactive` to skip confirmation) | Yes |
 | `lab task queue <id>` | Queue task on compute provider (`-m/--description` for a markdown run note; `-p/--param key=value` to override task parameters per run; required for agents, see "Always write a run description") | Yes |

.agents/skills/transformerlab-cli/references/commands.md

@@ -120,6 +120,25 @@ Add a new task from a local directory containing `task.yaml`, or from a Git repo
 
 **JSON output:** Returns the created task object.
 
+### `task edit <task_id>`
+
+Update an existing task on the server. Three modes:
+
+| Option | Description |
+|---|---|
+| (no flag) | Interactive — fetches current `task.yaml`, opens `$EDITOR`, validates and PUTs it back. YAML-only. |
+| `--from-file <path>` | Replace **only** `task.yaml` from the given path. Leaves `main.py` and other attachments untouched on the server. |
+| `--from-dir <path>` | Replace `task.yaml` **and** sibling files (e.g. `main.py`, configs). Zips the directory, uploads, and applies. **Use this whenever the task's `run` references a script file you've also modified — `--from-file` alone will desync the YAML from the script.** |
+| `--no-interactive` | Skip confirmation prompt (required in automated contexts) |
+| `--dry-run` | With `--from-dir`, preview the upload without submitting |
+| `--timeout <seconds>` | Request timeout for fetch/validate/save (default: 300) |
+
+`--from-file` and `--from-dir` are mutually exclusive.
+
+### `task upload <task_id> <path>`
+
+Upload additional files (or a whole directory) into an existing task without touching `task.yaml`. Useful when you want to add an attachment to a task whose YAML is already correct.
+
 ### `task delete <task_id>`
 
 Delete a task by ID.

api/transformerlab/services/compute_provider/launch_template.py

@@ -598,6 +598,13 @@ async def launch_template_on_provider(
         if await storage.isdir(task_src):
             workspace_job_dir = await get_job_dir(job_id, request.experiment_id)
             await copy_task_files_to_dir(task_src, workspace_job_dir)
+            # The local provider runs the user command with cwd=local_provider_job_dir,
+            # which is a separate directory from workspace_job_dir. Copy task files
+            # there too so commands like `python main.py` resolve relative to cwd.
+            if provider.type == ProviderType.LOCAL.value:
+                local_job_dir = provider_config_dict.get("workspace_dir")
+                if local_job_dir:
+                    await copy_task_files_to_dir(task_src, local_job_dir)
 
     job_data = {
         "task_name": request.task_name,

cli/src/transformerlab_cli/commands/task.py

@@ -724,11 +724,42 @@ def command_task_validate(
 def command_task_edit(
     task_id: str = typer.Argument(..., help="Task ID to update"),
     from_file: str | None = typer.Option(None, "--from-file", help="Path to task.yaml to apply directly"),
+    from_dir: str | None = typer.Option(
+        None,
+        "--from-dir",
+        help="Path to a directory containing task.yaml (and any attachments) to fully replace task contents",
+    ),
+    dry_run: bool = typer.Option(
+        False,
+        "--dry-run",
+        help="Preview the task update without submitting it (only applies with --from-dir)",
+    ),
     no_interactive: bool = typer.Option(False, "--no-interactive", help="Skip confirmation prompt"),
     timeout: int = typer.Option(300, "--timeout", help="Request timeout in seconds for fetch/validate/save"),
 ):
-    """Edit an existing task's task.yaml (interactive by default)."""
+    """Edit an existing task's task.yaml (interactive by default).
+
+    Use --from-file to replace only task.yaml, or --from-dir to replace task.yaml
+    plus any sibling files (e.g. main.py) in the task directory.
+    """
+    if from_file and from_dir:
+        console.print("[error]Error:[/error] --from-file and --from-dir are mutually exclusive")
+        raise typer.Exit(1)
+    if dry_run and not from_dir:
+        console.print("[warning]Warning:[/warning] --dry-run only applies with --from-dir; ignoring.")
+
     current_experiment = require_current_experiment()
+
+    if from_dir:
+        edit_task_from_directory(
+            task_id=task_id,
+            task_directory_path=from_dir,
+            experiment_id=current_experiment,
+            dry_run=dry_run,
+            interactive=not no_interactive,
+        )
+        return
+
     edit_task_yaml(
         task_id=task_id,
         experiment_id=current_experiment,

cli/tests/commands/test_task.py

@@ -299,6 +299,45 @@ def test_task_edit_updates_yaml_from_file(_mock_exp, _mock_get, mock_put, _mock_
     assert submit_path == "/experiment/exp1/task/t1/yaml"
 
 
+@patch("transformerlab_cli.commands.task.api.post_text", return_value=_mock_resp({"valid": True}))
+@patch("transformerlab_cli.commands.task.api.put", return_value=_mock_resp({"received": [0]}))
+@patch("transformerlab_cli.commands.task.api.get", return_value=_mock_resp({"received": []}))
+@patch("transformerlab_cli.commands.task.api.post_json")
+@patch("transformerlab_cli.commands.task.require_current_experiment", return_value="exp1")
+def test_task_edit_from_dir_uploads_directory_zip(_mock_exp, mock_post_json, _mock_get, _mock_put, _mock_post_text):
+    """`lab task edit --from-dir` zips the directory and POSTs to the /edit endpoint."""
+    mock_post_json.side_effect = [
+        _mock_resp({"upload_id": "up-1", "chunk_size": 64 * 1024 * 1024}),
+        _mock_resp({"status": "ok"}),
+        _mock_resp({"id": "t1"}),
+    ]
+    with runner.isolated_filesystem():
+        task_dir = Path("task")
+        task_dir.mkdir()
+        (task_dir / "task.yaml").write_text("name: demo\nrun: python main.py\n", encoding="utf-8")
+        (task_dir / "main.py").write_text("print('hi')\n", encoding="utf-8")
+        result = runner.invoke(app, ["task", "edit", "t1", "--from-dir", str(task_dir), "--no-interactive"])
+    assert result.exit_code == 0, result.output
+    submit_path = mock_post_json.call_args_list[-1].args[0]
+    assert submit_path == "/experiment/exp1/task/t1/edit?upload_id=up-1"
+
+
+def test_task_edit_rejects_from_file_and_from_dir_together():
+    """`lab task edit --from-file ... --from-dir ...` is rejected as mutually exclusive."""
+    with runner.isolated_filesystem():
+        task_yaml = Path("task.yaml")
+        task_yaml.write_text("name: demo\n", encoding="utf-8")
+        task_dir = Path("task")
+        task_dir.mkdir()
+        (task_dir / "task.yaml").write_text("name: demo\n", encoding="utf-8")
+        result = runner.invoke(
+            app,
+            ["task", "edit", "t1", "--from-file", str(task_yaml), "--from-dir", str(task_dir)],
+        )
+    assert result.exit_code != 0
+    assert "mutually exclusive" in strip_ansi(result.output)
+
+
 @patch("transformerlab_cli.commands.task.api.put", return_value=_mock_resp({"received": [0]}))
 @patch("transformerlab_cli.commands.task.api.get", return_value=_mock_resp({"received": []}))
 @patch("transformerlab_cli.commands.task.api.post_json")

package-lock.json

@@ -25,7 +25,7 @@
       "url": "https://github.com/dadmobile"
     }
   ],
-  "version": "0.37.0",
+  "version": "0.37.2",
   "scripts": {
     "build": "dotenv -e .env -- cross-env NODE_ENV=production tsx node_modules/webpack/bin/webpack.js --config ./.erb/configs/webpack.config.cloud.prod.ts",
     "build:multiuser": "cross-env NODE_ENV=production tsx node_modules/webpack/bin/webpack.js --config ./.erb/configs/webpack.config.cloud.prod.ts",

package.json

@@ -1,8 +1,7 @@
 import { test, expect, Page } from '@playwright/test';
 import { login, selectFirstExperiment } from './helpers';
 
-const GITHUB_REPO_URL =
-  'https://github.com/transformerlab/transformerlab-examples';
+const GITHUB_REPO_URL = 'https://github.com/transformerlab/transformerlab-app';
 const GITHUB_SUBDIR =
   'api/transformerlab/galleries/examples/demo-generate-task';
 
@@ -32,7 +31,7 @@ async function replaceTaskNameInMonaco(page: Page, taskName: string) {
 }
 
 test.describe('Dataset Generation Task From GitHub', () => {
-  test.setTimeout(180_000);
+  test.setTimeout(420_000);
 
   test('create task from GitHub, run local job, and require COMPLETE - 100%', async ({
     page,
@@ -112,7 +111,7 @@ test.describe('Dataset Generation Task From GitHub', () => {
         {
           message:
             'Expected job to reach COMPLETE - 100% (and never FAILED/COMPLETE - 0%)',
-          timeout: 120000,
+          timeout: 300000,
           intervals: [2000],
         },
       )

test/playwright/dataset-generate-task-github.spec.ts

@@ -31,7 +31,7 @@ async function replaceMonacoContents(page: Page, contents: string) {
 }
 
 test.describe('Dataset Generation Task', () => {
-  test.setTimeout(180_000);
+  test.setTimeout(420_000);
 
   test('create blank task, edit task.yaml, run local job, save dataset to registry, verify in Datasets page', async ({
     page,
@@ -118,7 +118,7 @@ run: "python demo-generate-task/fake_generate.py"
         {
           message:
             'Expected dataset job to reach COMPLETE - 100% (and never FAILED/COMPLETE - 0%)',
-          timeout: 120000,
+          timeout: 300000,
           intervals: [2000],
         },
       )

test/playwright/dataset-generate-task.spec.ts

@@ -32,7 +32,7 @@ async function replaceTaskNameInMonaco(page: Page, taskName: string) {
  */
 
 test.describe('Hello World Task', () => {
-  test.setTimeout(120_000);
+  test.setTimeout(300_000);
 
   test('create blank task, run on local provider, verify output in Machine Logs', async ({
     page,
@@ -116,7 +116,7 @@ test.describe('Hello World Task', () => {
 
     // ── Step 5: Wait for the job to complete ──
     await expect(queuedJobRow.getByText('COMPLETE')).toBeVisible({
-      timeout: 60000,
+      timeout: 180000,
     });
 
     // ── Step 6: Open the Output modal and verify "hello" in Machine Logs ──