Fix SSL certificate verification issue in provider data fetching

docs/cli.md

@@ -0,0 +1,120 @@
+# CrewAI CLI Documentation
+
+The CrewAI Command Line Interface (CLI) provides tools for creating, managing, and running CrewAI projects.
+
+## Installation
+
+The CLI is automatically installed when you install the CrewAI package:
+
+```bash
+pip install crewai
+```
+
+## Available Commands
+
+### Create Command
+
+The `create` command allows you to create new crews or flows.
+
+```bash
+crewai create [TYPE] [NAME] [OPTIONS]
+```
+
+#### Arguments
+
+- `TYPE`: Type of project to create. Must be either `crew` or `flow`.
+- `NAME`: Name of the project to create.
+
+#### Options
+
+- `--provider`: The provider to use for the crew.
+- `--skip_provider`: Skip provider validation.
+- `--skip_ssl_verify`: Skip SSL certificate verification when fetching provider data (not secure).
+
+#### Examples
+
+Create a new crew:
+
+```bash
+crewai create crew my_crew
+```
+
+Create a new crew with a specific provider:
+
+```bash
+crewai create crew my_crew --provider openai
+```
+
+Create a new crew and skip SSL certificate verification (useful in environments with self-signed certificates):
+
+```bash
+crewai create crew my_crew --skip_ssl_verify
+```
+
+> **Warning**: Using the `--skip_ssl_verify` flag is not recommended in production environments as it bypasses SSL certificate verification, which can expose your system to security risks. Only use this flag in development environments or when you understand the security implications.
+
+Create a new flow:
+
+```bash
+crewai create flow my_flow
+```
+
+### Run Command
+
+The `run` command executes your crew.
+
+```bash
+crewai run
+```
+
+### Train Command
+
+The `train` command trains your crew.
+
+```bash
+crewai train [OPTIONS]
+```
+
+#### Options
+
+- `-n, --n_iterations`: Number of iterations to train the crew (default: 5).
+- `-f, --filename`: Path to a custom file for training (default: "trained_agents_data.pkl").
+
+### Reset Memories Command
+
+The `reset_memories` command resets the crew memories.
+
+```bash
+crewai reset_memories [OPTIONS]
+```
+
+#### Options
+
+- `-l, --long`: Reset LONG TERM memory.
+- `-s, --short`: Reset SHORT TERM memory.
+- `-e, --entities`: Reset ENTITIES memory.
+- `-kn, --knowledge`: Reset KNOWLEDGE storage.
+- `-k, --kickoff-outputs`: Reset LATEST KICKOFF TASK OUTPUTS.
+- `-a, --all`: Reset ALL memories.
+
+### Other Commands
+
+- `version`: Show the installed version of crewai.
+- `replay`: Replay the crew execution from a specific task.
+- `log_tasks_outputs`: Retrieve your latest crew.kickoff() task outputs.
+- `test`: Test the crew and evaluate the results.
+- `install`: Install the Crew.
+- `update`: Update the pyproject.toml of the Crew project to use uv.
+- `signup`: Sign Up/Login to CrewAI+.
+- `login`: Sign Up/Login to CrewAI+.
+- `chat`: Start a conversation with the Crew.
+
+## Security Considerations
+
+When using the CrewAI CLI, be aware of the following security considerations:
+
+1. **API Keys**: Store your API keys securely in environment variables or a `.env` file. Never hardcode them in your scripts.
+
+2. **SSL Verification**: The `--skip_ssl_verify` flag bypasses SSL certificate verification, which can expose your system to security risks. Only use this flag in development environments or when you understand the security implications.
+
+3. **Provider Data**: When fetching provider data, ensure that you're using a secure connection. The CLI will display a warning when SSL verification is disabled.

src/crewai/cli/cli.py

@@ -1,6 +1,5 @@
-import os
 from importlib.metadata import version as get_version
-from typing import Optional, Tuple
+from typing import Optional
 
 import click
 
@@ -37,10 +36,11 @@ def crewai():
 @click.argument("name")
 @click.option("--provider", type=str, help="The provider to use for the crew")
 @click.option("--skip_provider", is_flag=True, help="Skip provider validation")
-def create(type, name, provider, skip_provider=False):
+@click.option("--skip_ssl_verify", is_flag=True, help="Skip SSL certificate verification (not secure)")
+def create(type, name, provider, skip_provider=False, skip_ssl_verify=False):
     """Create a new crew, or flow."""
     if type == "crew":
-        create_crew(name, provider, skip_provider)
+        create_crew(name, provider, skip_provider, skip_ssl_verify=skip_ssl_verify)
     elif type == "flow":
         create_flow(name)
     else:

src/crewai/cli/create_crew.py

@@ -89,12 +89,12 @@ def copy_template_files(folder_path, name, class_name, parent_folder):
             copy_template(src_file, dst_file, name, class_name, folder_path.name)
 
 
-def create_crew(name, provider=None, skip_provider=False, parent_folder=None):
+def create_crew(name, provider=None, skip_provider=False, parent_folder=None, skip_ssl_verify=False):
     folder_path, folder_name, class_name = create_folder_structure(name, parent_folder)
     env_vars = load_env_vars(folder_path)
     if not skip_provider:
         if not provider:
-            provider_models = get_provider_data()
+            provider_models = get_provider_data(skip_ssl_verify)
             if not provider_models:
                 return
 
@@ -114,7 +114,7 @@ def create_crew(name, provider=None, skip_provider=False, parent_folder=None):
                 click.secho("Keeping existing provider configuration.", fg="yellow")
                 return
 
-        provider_models = get_provider_data()
+        provider_models = get_provider_data(skip_ssl_verify)
         if not provider_models:
             return
 

src/crewai/cli/provider.py

@@ -106,13 +106,14 @@ def select_model(provider, provider_models):
     return selected_model
 
 
-def load_provider_data(cache_file, cache_expiry):
+def load_provider_data(cache_file, cache_expiry, skip_ssl_verify=False):
     """
     Loads provider data from a cache file if it exists and is not expired. If the cache is expired or corrupted, it fetches the data from the web.
 
     Args:
     - cache_file (Path): The path to the cache file.
     - cache_expiry (int): The cache expiry time in seconds.
+    - skip_ssl_verify (bool): Whether to skip SSL certificate verification.
 
     Returns:
     - dict or None: The loaded provider data or None if the operation fails.
@@ -133,7 +134,7 @@ def load_provider_data(cache_file, cache_expiry):
             "Cache expired or not found. Fetching provider data from the web...",
             fg="cyan",
         )
-    return fetch_provider_data(cache_file)
+    return fetch_provider_data(cache_file, skip_ssl_verify)
 
 
 def read_cache_file(cache_file):
@@ -147,34 +148,65 @@ def read_cache_file(cache_file):
     - dict or None: The JSON content of the cache file or None if the JSON is invalid.
     """
     try:
+        if not cache_file.exists():
+            return None
         with open(cache_file, "r") as f:
-            return json.load(f)
-    except json.JSONDecodeError:
+            data = json.load(f)
+            if not isinstance(data, dict):
+                click.secho("Invalid cache file format", fg="yellow")
+                return None
+            return data
+    except json.JSONDecodeError as e:
+        click.secho(f"Error parsing cache file: {str(e)}", fg="yellow")
+        return None
+    except OSError as e:
+        click.secho(f"Error reading cache file: {str(e)}", fg="yellow")
         return None
 
 
-def fetch_provider_data(cache_file):
+def fetch_provider_data(cache_file, skip_ssl_verify=False):
     """
     Fetches provider data from a specified URL and caches it to a file.
 
     Args:
     - cache_file (Path): The path to the cache file.
+    - skip_ssl_verify (bool): Whether to skip SSL certificate verification.
 
     Returns:
     - dict or None: The fetched provider data or None if the operation fails.
     """
     try:
-        response = requests.get(JSON_URL, stream=True, timeout=60)
+        if skip_ssl_verify:
+            click.secho(
+                "Warning: SSL certificate verification is disabled. This is not secure!",
+                fg="yellow",
+            )
+        response = requests.get(JSON_URL, stream=True, timeout=60, verify=not skip_ssl_verify)
         response.raise_for_status()
         data = download_data(response)
         with open(cache_file, "w") as f:
             json.dump(data, f)
         return data
+    except requests.Timeout:
+        click.secho("Timeout while fetching provider data", fg="red")
+        return None
+    except requests.SSLError as e:
+        click.secho(f"SSL verification failed: {str(e)}", fg="red")
+        if not skip_ssl_verify:
+            click.secho(
+                "You can bypass SSL verification with --skip_ssl_verify flag (not secure)",
+                fg="yellow",
+            )
+        return None
     except requests.RequestException as e:
-        click.secho(f"Error fetching provider data: {e}", fg="red")
+        click.secho(f"Error fetching provider data: {str(e)}", fg="red")
+        return None
     except json.JSONDecodeError:
         click.secho("Error parsing provider data. Invalid JSON format.", fg="red")
-    return None
+        return None
+    except OSError as e:
+        click.secho(f"Error writing to cache file: {str(e)}", fg="red")
+        return None
 
 
 def download_data(response):
@@ -201,10 +233,13 @@ def download_data(response):
     return json.loads(data_content.decode("utf-8"))
 
 
-def get_provider_data():
+def get_provider_data(skip_ssl_verify=False):
     """
     Retrieves provider data from a cache file, filters out models based on provider criteria, and returns a dictionary of providers mapped to their models.
 
+    Args:
+    - skip_ssl_verify (bool): Whether to skip SSL certificate verification.
+
     Returns:
     - dict or None: A dictionary of providers mapped to their models or None if the operation fails.
     """
@@ -213,7 +248,7 @@ def get_provider_data():
     cache_file = cache_dir / "provider_cache.json"
     cache_expiry = 24 * 3600
 
-    data = load_provider_data(cache_file, cache_expiry)
+    data = load_provider_data(cache_file, cache_expiry, skip_ssl_verify)
     if not data:
         return None