Merge pull request #1815 from transformerlab/add/gcs-sa-key-setup

add: prompt for GCP service account key path during setup

Author: greninja

api/transformerlab/services/compute_provider/launch_credentials.py

@@ -66,15 +66,19 @@ def escape_bash(s: str) -> str:
     return setup_script
 
 
-def generate_gcp_credentials_setup(service_account_json: str, credentials_path: Optional[str] = None) -> str:
-    target_path = credentials_path or "~/.config/gcloud/application_default_credentials.json"
-    encoded = base64.b64encode(service_account_json.encode()).decode()
+def generate_gcp_credentials_setup(sa_json_path: str) -> str:
+    """Read service account JSON from *sa_json_path* and emit a setup script that writes
+    it to the standard ADC location so GCP client libraries pick it up automatically."""
+    sa_json_path = os.path.expanduser(sa_json_path)
+    with open(sa_json_path, "r", encoding="utf-8") as f:
+        encoded = base64.b64encode(f.read().encode()).decode()
+    adc_path = "~/.config/gcloud/application_default_credentials.json"
     setup_script = (
         "echo 'Setting up GCP service account credentials...'; "
-        f"mkdir -p $(dirname {target_path}); "
-        f"echo '{encoded}' | base64 -d > {target_path}; "
-        f"chmod 600 {target_path}; "
-        f"export GOOGLE_APPLICATION_CREDENTIALS={target_path}; "
+        "mkdir -p ~/.config/gcloud; "
+        f"echo '{encoded}' | base64 -d > {adc_path}; "
+        f"chmod 600 {adc_path}; "
+        f"export GOOGLE_APPLICATION_CREDENTIALS={adc_path}; "
         "echo 'GCP credentials configured successfully'"
     )
     return setup_script

api/transformerlab/services/compute_provider/launch_sweep.py

@@ -254,9 +254,9 @@ async def launch_sweep_jobs(
                             if aws_credentials_dir:
                                 env_vars["AWS_SHARED_CREDENTIALS_FILE"] = f"{aws_credentials_dir}/credentials"
                     elif STORAGE_PROVIDER == "gcp":
-                        gcp_sa_json = os.getenv("TFL_GCP_SERVICE_ACCOUNT_JSON")
-                        if gcp_sa_json:
-                            gcp_setup = generate_gcp_credentials_setup(gcp_sa_json)
+                        gcp_sa_json_path = os.getenv("TFL_GCP_SERVICE_ACCOUNT_JSON_PATH")
+                        if gcp_sa_json_path:
+                            gcp_setup = generate_gcp_credentials_setup(gcp_sa_json_path)
                             setup_commands.append(gcp_setup)
                     elif STORAGE_PROVIDER == "azure":
                         azure_connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING")

api/transformerlab/services/compute_provider/launch_template.py

@@ -231,9 +231,9 @@ async def launch_template_on_provider(
                 if aws_credentials_dir:
                     env_vars["AWS_SHARED_CREDENTIALS_FILE"] = f"{aws_credentials_dir}/credentials"
         elif STORAGE_PROVIDER == "gcp":
-            gcp_sa_json = os.getenv("TFL_GCP_SERVICE_ACCOUNT_JSON")
-            if gcp_sa_json:
-                gcp_setup = generate_gcp_credentials_setup(gcp_sa_json)
+            gcp_sa_json_path = os.getenv("TFL_GCP_SERVICE_ACCOUNT_JSON_PATH")
+            if gcp_sa_json_path:
+                gcp_setup = generate_gcp_credentials_setup(gcp_sa_json_path)
                 setup_commands.append(gcp_setup)
         elif STORAGE_PROVIDER == "azure":
             azure_connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING")

cli/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "transformerlab-cli"
-version = "0.0.19"
+version = "0.0.20"
 description = "Transformer Lab CLI"
 requires-python = ">=3.10"
 authors = [{ name = "Transformer Lab", email = "hello@transformerlab.ai" }]

cli/src/transformerlab_cli/commands/server.py

@@ -1,3 +1,4 @@
+import json
 import os
 import secrets
 import shutil
@@ -173,6 +174,40 @@ def _prompt_storage(existing: dict[str, str]) -> dict[str, str]:
             if project.strip():
                 env["GCP_PROJECT"] = project.strip()
             env["REMOTE_WORKSPACE_HOST"] = "gcp"
+
+            default_sa_path = existing.get("TFL_GCP_SERVICE_ACCOUNT_JSON_PATH", "")
+            sa_path = typer.prompt(
+                "Path to service account JSON key file (for remote job launches, press Enter to skip)",
+                default=default_sa_path,
+                show_default=False,
+            ).strip()
+            if sa_path:
+                sa_path = os.path.expanduser(sa_path)
+                if not os.path.isfile(sa_path):
+                    console.print(
+                        f"[error]File not found: {sa_path} — skipping service account key setup.[/error]"
+                        "\n[bold error]WARNING: Remote Job (using GCP for storage) launches will fail without a valid service account key.[/bold error]"
+                    )
+                else:
+                    try:
+                        with open(sa_path, "r", encoding="utf-8") as _f:
+                            json.load(_f)
+                        env["TFL_GCP_SERVICE_ACCOUNT_JSON_PATH"] = sa_path
+                    except json.JSONDecodeError:
+                        console.print(
+                            "[error]File is not valid JSON — skipping service account key setup.[/error]"
+                            "\n[bold error]WARNING: Remote Job (using GCP for storage) launches will fail without a valid service account key.[/bold error]"
+                        )
+                    except OSError as e:
+                        console.print(
+                            f"[error]Could not read file: {e} — skipping service account key setup.[/error]"
+                            "\n[bold error]WARNING: Remote Job (using GCP for storage) launches will fail without a valid service account key.[/bold error]"
+                        )
+            else:
+                console.print(
+                    "[bold error]WARNING: No service account key provided. Remote Job (using GCP for storage) launches will fail.[/bold error]"
+                    "\n[dim]You can re-run 'lab server init' to configure this later.[/dim]"
+                )
         elif provider == "azure":
             console.print("\n[info]Azure Blob Storage authentication:[/info]")
             use_conn_string = typer.confirm("Use a connection string?", default=True)