Alow upload of datasets and graphs to the cloud service (#544)

* Alow upload of datasets and graphs to the cloud service

Signed-off-by: Luke Hinds <lukehinds@gmail.com>

* Fix tests for upload url missing dataset path

Signed-off-by: Luke Hinds <lukehinds@gmail.com>

---------

Signed-off-by: Luke Hinds <lukehinds@gmail.com>

Author: lukehinds

deepfabric/cli.py

@@ -745,6 +745,224 @@ def upload_kaggle(
         sys.exit(1)
 
 
+# DeepFabric Cloud upload command group
+@click.group()
+def upload() -> None:
+    """Upload datasets and graphs to DeepFabric Cloud."""
+    pass
+
+
+def _upload_to_cloud(
+    file: str,
+    resource_type: Literal["dataset", "graph"],
+    handle: str | None,
+    name: str | None,
+    description: str | None,
+    tags: list[str] | None,
+    config_file: str | None,
+) -> None:
+    """Shared helper for uploading datasets and graphs to DeepFabric Cloud.
+
+    Args:
+        file: Path to the file to upload
+        resource_type: Either "dataset" or "graph"
+        handle: Resource handle (e.g., username/resource-name)
+        name: Display name for the resource
+        description: Description for the resource
+        tags: Tags for the resource (only used for datasets)
+        config_file: Path to config file with upload settings
+    """
+    # Lazy imports to avoid slow startup
+    import httpx  # noqa: PLC0415
+
+    from .auth import DEFAULT_API_URL  # noqa: PLC0415
+    from .cloud_upload import (  # noqa: PLC0415
+        _get_user_friendly_error,
+        build_urls,
+        derive_frontend_url,
+        derive_name_and_slug,
+        ensure_authenticated,
+        get_current_user,
+        upload_dataset,
+        upload_topic_graph,
+    )
+
+    tui = get_tui()
+    config_key = resource_type  # "dataset" or "graph"
+    url_resource_type = "datasets" if resource_type == "dataset" else "graphs"
+
+    # Load handle from config if not provided via CLI
+    final_handle = handle
+    final_description = description or ""
+    final_tags = list(tags) if tags else []
+
+    if config_file:
+        config = DeepFabricConfig.from_yaml(config_file)
+        cloud_config = config.get_deepfabric_cloud_config()
+        if not final_handle:
+            final_handle = cloud_config.get(config_key)
+        if not description and cloud_config.get("description"):
+            final_description = cloud_config.get("description", "")
+        if resource_type == "dataset" and not tags and cloud_config.get("tags"):
+            final_tags = cloud_config.get("tags", [])
+
+    # Ensure authenticated
+    if not ensure_authenticated(DEFAULT_API_URL, headless=False):
+        tui.error("Authentication required. Run 'deepfabric auth login' first.")
+        sys.exit(1)
+
+    # Derive name and slug from filename if not provided
+    default_name, default_slug = derive_name_and_slug(file)
+    final_name = name or default_name
+
+    # Use slug from handle if provided, otherwise use derived slug
+    if final_handle and "/" in final_handle:
+        final_slug = final_handle.split("/")[-1]
+    else:
+        final_slug = final_handle or default_slug
+
+    tui.info(f"Uploading {resource_type} '{final_name}'...")
+
+    try:
+        # Call the appropriate upload function
+        if resource_type == "dataset":
+            result = upload_dataset(
+                dataset_path=file,
+                name=final_name,
+                slug=final_slug,
+                description=final_description,
+                tags=final_tags,
+                api_url=DEFAULT_API_URL,
+            )
+            resource_id = result.get("dataset_id") or result.get("id")
+        else:
+            result = upload_topic_graph(
+                graph_path=file,
+                name=final_name,
+                description=final_description,
+                slug=final_slug,
+                api_url=DEFAULT_API_URL,
+            )
+            resource_id = result.get("id")
+
+        # Display success message
+        tui.success(f"{resource_type.capitalize()} '{final_name}' uploaded successfully!")
+
+        # Display URL if available
+        if resource_id:
+            user_info = get_current_user(DEFAULT_API_URL)
+            username = user_info.get("username") if user_info else None
+            frontend_url = derive_frontend_url(DEFAULT_API_URL)
+            public_url, internal_url = build_urls(
+                url_resource_type, resource_id, final_slug, username, frontend_url
+            )
+            tui.info(f"View at: {public_url or internal_url}")
+
+    except httpx.HTTPStatusError as e:
+        error_msg = _get_user_friendly_error(e)
+        if "already exists" in error_msg.lower():
+            tui.error(
+                f"A {resource_type} with slug '{final_slug}' already exists. "
+                "Use a different --handle value."
+            )
+        else:
+            tui.error(f"Error uploading {resource_type}: {error_msg}")
+        sys.exit(1)
+    except Exception as e:
+        tui.error(f"Error uploading {resource_type}: {str(e)}")
+        sys.exit(1)
+
+
+@upload.command("dataset")
+@click.argument("file", type=click.Path(exists=True))
+@click.option("--handle", help="Dataset handle (e.g., username/dataset-name)")
+@click.option("--name", help="Display name for the dataset")
+@click.option("--description", help="Description for the dataset")
+@click.option(
+    "--tags", multiple=True, help="Tags for the dataset (can be specified multiple times)"
+)
+@click.option(
+    "--config",
+    "config_file",
+    type=click.Path(exists=True),
+    help="Config file with upload settings",
+)
+def upload_dataset_cmd(
+    file: str,
+    handle: str | None,
+    name: str | None,
+    description: str | None,
+    tags: tuple[str, ...],
+    config_file: str | None,
+) -> None:
+    """Upload a dataset to DeepFabric Cloud.
+
+    FILE is the path to the JSONL dataset file.
+
+    Examples:
+
+        deepfabric upload dataset my-dataset.jsonl --handle myuser/my-dataset
+
+        deepfabric upload dataset output.jsonl --config config.yaml
+    """
+    trace(
+        "cli_upload_dataset",
+        {"has_config": config_file is not None, "has_handle": handle is not None},
+    )
+    _upload_to_cloud(
+        file=file,
+        resource_type="dataset",
+        handle=handle,
+        name=name,
+        description=description,
+        tags=list(tags) if tags else None,
+        config_file=config_file,
+    )
+
+
+@upload.command("graph")
+@click.argument("file", type=click.Path(exists=True))
+@click.option("--handle", help="Graph handle (e.g., username/graph-name)")
+@click.option("--name", help="Display name for the graph")
+@click.option("--description", help="Description for the graph")
+@click.option(
+    "--config",
+    "config_file",
+    type=click.Path(exists=True),
+    help="Config file with upload settings",
+)
+def upload_graph_cmd(
+    file: str,
+    handle: str | None,
+    name: str | None,
+    description: str | None,
+    config_file: str | None,
+) -> None:
+    """Upload a topic graph to DeepFabric Cloud.
+
+    FILE is the path to the JSON graph file.
+
+    Examples:
+
+        deepfabric upload graph topic_graph.json --handle myuser/my-graph
+
+        deepfabric upload graph graph.json --config config.yaml
+    """
+    trace(
+        "cli_upload_graph",
+        {"has_config": config_file is not None, "has_handle": handle is not None},
+    )
+    _upload_to_cloud(
+        file=file,
+        resource_type="graph",
+        handle=handle,
+        name=name,
+        description=description,
+        tags=None,
+        config_file=config_file,
+    )
+
+
 @cli.command()
 @click.argument("graph_file", type=click.Path(exists=True))
 @click.option(
@@ -1152,10 +1370,11 @@ def evaluate(
         handle_error(click.get_current_context(), e)
 
 
-# Register the auth command group
+# Register the auth and upload command groups
 # EXPERIMENTAL: Only enable cloud features if explicitly opted in
 if get_bool_env("EXPERIMENTAL_DF"):
     cli.add_command(auth_group)
+    cli.add_command(upload)
 
 
 @cli.command("import-tools")

deepfabric/cloud_upload.py

@@ -359,7 +359,7 @@ def build_urls(
 
     if username:
         if resource_type == "datasets":
-            public_url = f"{frontend_url}/{username}/{slug}"
+            public_url = f"{frontend_url}/dataset/{username}/{slug}"
         else:
             public_url = f"{frontend_url}/graphs/{username}/{slug}"
     else:

deepfabric/config.py

@@ -345,6 +345,27 @@ class KaggleConfig(BaseModel):
     version_notes: str | None = Field(None, description="Version notes for dataset update")
 
 
+class DeepFabricCloudConfig(BaseModel):
+    """Configuration for DeepFabric Cloud integration."""
+
+    graph: str | None = Field(
+        default=None,
+        description="DeepFabric Cloud graph handle (e.g., username/graph-name)",
+    )
+    dataset: str | None = Field(
+        default=None,
+        description="DeepFabric Cloud dataset handle (e.g., username/dataset-name)",
+    )
+    description: str | None = Field(
+        default=None,
+        description="Description for uploaded resources",
+    )
+    tags: list[str] = Field(
+        default_factory=list,
+        description="Tags for uploaded resources",
+    )
+
+
 class EvaluationConfig(BaseModel):
     """Configuration for model evaluation."""
 
@@ -456,6 +477,9 @@ class DeepFabricConfig(BaseModel):
     evaluation: EvaluationConfig | None = Field(None, description="Evaluation configuration")
     huggingface: HuggingFaceConfig | None = Field(None, description="Hugging Face configuration")
     kaggle: KaggleConfig | None = Field(None, description="Kaggle configuration")
+    deepfabric_cloud: DeepFabricCloudConfig | None = Field(
+        None, description="DeepFabric Cloud configuration"
+    )
 
     @classmethod
     def _detect_old_format(cls, config_dict: dict) -> bool:
@@ -662,6 +686,10 @@ def get_kaggle_config(self) -> dict:
         """Get Kaggle configuration."""
         return self.kaggle.model_dump() if self.kaggle else {}
 
+    def get_deepfabric_cloud_config(self) -> dict:
+        """Get DeepFabric Cloud configuration."""
+        return self.deepfabric_cloud.model_dump() if self.deepfabric_cloud else {}
+
     def get_configured_providers(self) -> set[str]:
         """Get the set of LLM providers configured in this config."""
         providers = set()

tests/unit/test_cloud_upload.py

@@ -69,7 +69,7 @@ def test_dataset_with_username(self):
             username="testuser",
             frontend_url="https://example.deepfabric.null",
         )
-        assert public_url == "https://example.deepfabric.null/testuser/my-dataset"
+        assert public_url == "https://example.deepfabric.null/dataset/testuser/my-dataset"
         assert internal_url == "https://example.deepfabric.null/datasets/abc123"
 
     def test_dataset_without_username(self):