Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion api/localprovider_pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ dependencies = [
"soundfile==0.13.1",
"tensorboardX==2.6.2.2",
"timm==1.0.15",
"transformerlab==0.1.37",
"transformerlab==0.1.38",
"transformerlab-inference==0.2.52",
"transformers==4.57.1",
"wandb==0.23.1",
Expand Down
2 changes: 1 addition & 1 deletion api/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ dependencies = [
"python-dotenv==1.1.0",
"python-multipart==0.0.20",
"sqlalchemy[asyncio]==2.0.40",
"transformerlab==0.1.37",
"transformerlab==0.1.38",
"transformerlab-inference==0.2.52",
"uvicorn==0.35.0",
"watchfiles==1.0.5",
Expand Down
23 changes: 0 additions & 23 deletions api/test/api/test_model_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,29 +65,6 @@ async def test_list_installed_models_non_hf_with_model_filename(monkeypatch):
assert "local_path" in result[0]


@pytest.mark.asyncio
async def test_list_installed_models_non_hf_directory_model(monkeypatch):
"""A non-HuggingFace directory-based model (no model_filename) with files is marked local."""
from transformerlab.services import model_service

model = make_model("MyOrg/dir-model", source="local", model_filename="")
monkeypatch.setattr(model_service.ModelService, "list_all", AsyncMock(return_value=[model]))
monkeypatch.setattr(model_service, "get_models_dir", AsyncMock(return_value="/models"))

monkeypatch.setattr(model_service.storage, "join", lambda *parts: "/".join(parts))
monkeypatch.setattr(model_service.storage, "exists", AsyncMock(return_value=True))
monkeypatch.setattr(model_service.storage, "isdir", AsyncMock(return_value=True))
monkeypatch.setattr(
model_service.storage,
"ls",
AsyncMock(return_value=["/models/dir-model/weights.bin", "/models/dir-model/index.json"]),
)

result = await model_service.list_installed_models()
assert len(result) == 1
assert result[0]["stored_in_filesystem"] is True


@pytest.mark.asyncio
async def test_list_installed_models_non_hf_directory_only_index(monkeypatch):
"""A non-HuggingFace directory with only index.json is NOT marked as local."""
Expand Down
6 changes: 3 additions & 3 deletions api/transformerlab/routers/experiment/task.py
Original file line number Diff line number Diff line change
Expand Up @@ -1166,10 +1166,10 @@ async def import_task_from_gallery(
interactive_type = gallery_entry.get("interactive_type") or gallery_entry.get("id") or "custom"
interactive_gallery_id = gallery_entry.get("id")

# Resolve task setup/command from the gallery entry's source:
# Resolve task setup/run from the gallery entry's source:
# 1. github_repo_url + github_repo_dir -> fetch task.yaml from GitHub
# 2. local_task_dir -> read task.yaml from local filesystem
# 3. inline setup/command fields on the gallery entry
# 3. inline setup/run fields on the gallery entry
github_repo_url = gallery_entry.get("github_repo_url")
github_repo_dir = gallery_entry.get("github_repo_dir")
github_repo_branch = gallery_entry.get("github_repo_branch")
Expand Down Expand Up @@ -1197,7 +1197,7 @@ async def import_task_from_gallery(
"plugin": "remote_orchestrator",
"experiment_id": experimentId,
"cluster_name": task_name,
"run": source_yaml_data.get("run", source_yaml_data.get("command", "")),
"run": source_yaml_data.get("run", ""),
"setup": source_yaml_data.get("setup", "") or gallery_entry.get("setup", ""),
"interactive_type": interactive_type,
"subtype": "interactive",
Expand Down
31 changes: 3 additions & 28 deletions api/transformerlab/services/model_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,7 @@


async def list_installed_models() -> list:
"""
TODO: Clean this up to remove legacy code.

Legacy function for getting a list of models from all sources.

Check both the filesystem and workspace for models.
"""
"""Get a list of installed models with filesystem metadata attached."""

# Use SDK to get all models from the filesystem
models = await ModelService.list_all()
Expand All @@ -37,7 +31,6 @@ async def list_installed_models() -> list:
# model_filename can be:
# - A filename (e.g., "model.gguf") for file-based models
# - "." for directory-based models (indicates the directory itself)
# - Empty string for legacy models (should be treated as directory-based)
model_filename = model.get("json_data", {}).get("model_filename", "")
is_huggingface = model.get("json_data", {}).get("source", "") == "huggingface"
has_model_filename = model_filename != ""
Expand All @@ -51,27 +44,12 @@ async def list_installed_models() -> list:
# Remove the Starting TransformerLab/ prefix to handle the save_transformerlab_model function
potential_path = storage.join(models_dir, secure_filename("/".join(model_id.split("/")[1:])))

# Check if model should be considered local:
# 1. If it has a model_filename set (and is not a HuggingFace model, OR is a HuggingFace model stored locally), OR
# 2. If the directory exists and has files other than index.json
# A model is considered local if model_filename is set (non-HF) or if the
# HF model has model_filename and the file/directory exists locally.
is_local_model = False
if not is_huggingface:
# For non-HuggingFace models, check if it has model_filename or files in directory
if has_model_filename:
is_local_model = True
elif await storage.exists(potential_path) and await storage.isdir(potential_path):
# Check if directory has files other than index.json
try:
files = await storage.ls(potential_path, detail=False)
# Extract basenames from full paths returned by storage.ls()
file_basenames = [posixpath.basename(f.rstrip("/")) for f in files]
# Filter out index.json and other metadata files
model_files = [f for f in file_basenames if f not in ["index.json", "_tlab_provenance.json"]]
if model_files:
is_local_model = True
except (OSError, PermissionError):
# If we can't read the directory, skip it
pass
elif is_huggingface and has_model_filename:
# For HuggingFace models, if they have a model_filename and the file/directory exists locally,
# treat them as stored locally (e.g., downloaded GGUF files)
Expand Down Expand Up @@ -128,9 +106,6 @@ async def list_installed_models() -> list:
elif model_filename:
# Other file-based models - append the filename
model["local_path"] = storage.join(model["local_path"], model_filename)
else:
# Legacy model without model_filename but with files - use directory path
model["local_path"] = model["local_path"]

return models

Expand Down
42 changes: 6 additions & 36 deletions api/transformerlab/services/task_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,30 +22,6 @@
DEFAULT_TASK_YAML = 'name: my-task\nresources:\n cpus: 2\n memory: 4\nrun: "echo hello"'


def _normalize_legacy_command(task: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
"""
Backward compatibility helper: some legacy tasks may still store their
entrypoint under the key "command" instead of "run". To avoid breaking
those when launching or exporting, expose "run" on read if it is missing
or empty but "command" is present. New code should only write "run".
"""
if not task:
return task

# If run is already set and truthy, do nothing.
if task.get("run"):
return task

legacy_command = task.get("command")
if not legacy_command:
return task

# Return a shallow copy with run populated so callers always see run.
normalized = dict(task)
normalized["run"] = legacy_command
return normalized


class TaskService:
"""Service for managing tasks using filesystem storage"""

Expand All @@ -54,35 +30,29 @@ def __init__(self):

async def task_get_all(self) -> List[Dict[str, Any]]:
"""Get all tasks from filesystem"""
tasks = await self.task_service.list_all()
return [_normalize_legacy_command(t) or t for t in tasks]
return await self.task_service.list_all()

async def task_get_by_id(self, task_id: str, experiment_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
"""Get a specific task by ID"""
task = await self.task_service.get_by_id(task_id, experiment_id=experiment_id)
return _normalize_legacy_command(task)
return await self.task_service.get_by_id(task_id, experiment_id=experiment_id)

async def task_get_by_type(self, task_type: str) -> List[Dict[str, Any]]:
"""Get all tasks of a specific type"""
tasks = await self.task_service.list_by_type(task_type)
return [_normalize_legacy_command(t) or t for t in tasks]
return await self.task_service.list_by_type(task_type)

async def task_get_by_experiment(self, experiment_id: str) -> List[Dict[str, Any]]:
"""Get all tasks for a specific experiment"""
tasks = await self.task_service.list_by_experiment(experiment_id)
return [_normalize_legacy_command(t) or t for t in tasks]
return await self.task_service.list_by_experiment(experiment_id)

async def task_get_by_type_in_experiment(self, task_type: str, experiment_id: str) -> List[Dict[str, Any]]:
"""Get all tasks of a specific type in a specific experiment"""
tasks = await self.task_service.list_by_type_in_experiment(task_type, experiment_id)
return [_normalize_legacy_command(t) or t for t in tasks]
return await self.task_service.list_by_type_in_experiment(task_type, experiment_id)

async def task_get_by_subtype_in_experiment(
self, experiment_id: str, subtype: str, task_type: Optional[str] = None
) -> List[Dict[str, Any]]:
"""Get all tasks for a specific experiment filtered by subtype and optionally by type"""
tasks = await self.task_service.list_by_subtype_in_experiment(experiment_id, subtype, task_type)
return [_normalize_legacy_command(t) or t for t in tasks]
return await self.task_service.list_by_subtype_in_experiment(experiment_id, subtype, task_type)

async def add_task(self, task_data: Dict[str, Any]) -> str:
"""Create a new task - all fields stored directly in JSON"""
Expand Down
2 changes: 1 addition & 1 deletion cli/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

[project]
name = "transformerlab-cli"
version = "0.0.50"
version = "0.0.51"
description = "Transformer Lab CLI"
requires-python = ">=3.10"
authors = [{ name = "Transformer Lab", email = "hello@transformerlab.ai" }]
Expand Down
8 changes: 7 additions & 1 deletion cli/src/transformerlab_cli/commands/task.py
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,13 @@ def _upload_path_to_server(path_to_upload: str) -> str:
console.print(f"[error]Error:[/error] Upload init failed ({init_resp.status_code})")
raise typer.Exit(1)

upload_id = init_resp.json()["upload_id"]
try:
upload_id = init_resp.json().get("upload_id")
except ValueError:
upload_id = None
if not upload_id:
console.print("[error]Error:[/error] Upload init returned no upload_id.")
raise typer.Exit(1)
status_resp = api.get(f"/upload/{upload_id}/status")
already_received: set[int] = set(
status_resp.json().get("received", []) if status_resp.status_code == 200 else []
Expand Down
2 changes: 1 addition & 1 deletion cli/uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion lab-sdk/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

[project]
name = "transformerlab"
version = "0.1.37"
version = "0.1.38"
description = "Python SDK for Transformer Lab"
readme = "README.md"
requires-python = ">=3.10"
Expand Down
Loading
Loading