full_dependencies_explorer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Full Dependencies Analysis\n",
    "This notebook explores the recursive dependencies exported in full_dataset_dependencies.json\n",
    "\n",
    "Requirements: install networkx, matplotlib and ipywidgets via pip\n",
    "\n",
    "## Important\n",
    "Install nbstripout via pip and run command: nbstripout --install to ensure that outputs from this workbook are not synced to git."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup and Load"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "\n",
    "INPUT_FILE_NAME = \"full_dataset_dependencies.json\"\n",
    "INPUT_DIR = \"Output\"\n",
    "\n",
    "# Load the direct dependencies data\n",
    "file_path = f\"{INPUT_DIR}/{INPUT_FILE_NAME}\"\n",
    "with open(file_path, 'r') as file:\n",
    "    full_dependencies = json.load(file)\n",
    "\n",
    "# Convert to a DataFrame\n",
    "dependency_list = []\n",
    "for dataset, dependencies in full_dependencies.items():\n",
    "    for dep in dependencies:\n",
    "        dependency_list.append({\"Dataset\": dataset, \"Dependency\": dep})\n",
    "\n",
    "df = pd.DataFrame(dependency_list)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Basic Data Exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Display the first few rows of the DataFrame\n",
    "print(df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DataFrame shape (number of rows and columns)\n",
    "print(f\"DataFrame Shape: {df.shape}\")\n",
    "\n",
    "# Column data types\n",
    "print(f\"Data Types:\\n{df.dtypes}\")\n",
    "\n",
    "# Check for missing values\n",
    "missing_values = df.isnull().sum()\n",
    "print(f\"Missing Values:\\n{missing_values}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unique Dataset Count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Unique Datasets and Dependencies\n",
    "unique_datasets = df['Dataset'].nunique()\n",
    "unique_dependencies = df['Dependency'].nunique()\n",
    "print(f\"Unique Datasets: {unique_datasets}\")\n",
    "print(f\"Unique Dependencies: {unique_dependencies}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Frequent dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dependency_freq = df['Dependency'].value_counts().head(40)\n",
    "\n",
    "plt.figure(figsize=(20, 5))\n",
    "dependency_freq.plot(kind='bar')\n",
    "plt.xlabel('Dependency')\n",
    "plt.ylabel('Count')\n",
    "plt.title('Dependency Counts (Top 40)')\n",
    "\n",
    "# Set x-axis tick labels and rotation\n",
    "plt.xticks(rotation=90)\n",
    "\n",
    "# Show the plot\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Frequency of dependencies\n",
    "dependency_freq = df['Dependency'].value_counts().head(20)\n",
    "print(\"Most frequent dependencies:\\n\", dependency_freq)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Datasets that have no dependents\n",
    "This is datasets that have no other datasets that rely on it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_datasets_with_no_dependents(full_dependencies):\n",
    "    all_datasets = set(full_dependencies.keys())\n",
    "    datasets_with_dependents = set()\n",
    "\n",
    "    # Identifying all datasets that are dependencies\n",
    "    for dependencies in full_dependencies.values():\n",
    "        datasets_with_dependents.update(dependencies)\n",
    "\n",
    "    # Datasets with no dependents are those in all_datasets but not in datasets_with_dependents\n",
    "    datasets_with_no_dependents = all_datasets - datasets_with_dependents\n",
    "\n",
    "    # Sort the list alphabetically\n",
    "    return sorted(list(datasets_with_no_dependents))\n",
    "\n",
    "# Use the function with the full_dependencies dictionary\n",
    "datasets_with_no_dependents = find_datasets_with_no_dependents(full_dependencies)\n",
    "\n",
    "print(f\"Datasets with no dependents: {len(datasets_with_no_dependents)} of {unique_datasets} total datasets ({len(datasets_with_no_dependents)/unique_datasets:.0%})\")\n",
    "for dataset in datasets_with_no_dependents:\n",
    "    print(dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Independent Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the variable independent_datasets\n",
    "independent_datasets = df[df['Dependency'].isnull()]\n",
    "\n",
    "dataset_list = []\n",
    "for dataset in independent_datasets['Dataset'].unique():\n",
    "    dataset_list.append(dataset)\n",
    "\n",
    "print(f\"Total Independent Datasets: {len(dataset_list)}\")\n",
    "\n",
    "for dataset in dataset_list:\n",
    "    print(dataset)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dependencies and Dependents for a specific dataset\n",
    "\n",
    "### Dependencies\n",
    "- **Dependencies** refer to other datasets that the selected dataset **relies on**. \n",
    "- In simple terms, dependencies are what a dataset 'needs'.\n",
    "\n",
    "### Dependents\n",
    "- **Dependents** are datasets that **rely on** the selected dataset.\n",
    "- In essence, dependents are what 'need' the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ipywidgets as widgets\n",
    "from ipywidgets import interact\n",
    "\n",
    "# Function to display dependencies and dependents of a selected dataset\n",
    "def display_dataset_dependencies(dataset_name):\n",
    "    # Dependencies\n",
    "    dependencies = full_dependencies.get(dataset_name, [])\n",
    "    print(f\"Total Dependencies: {len(dependencies)}\")\n",
    "    if dependencies:\n",
    "        print(\"Dependencies - datasets that are relied upon for the selected dataset to run:\")\n",
    "        for dep in dependencies:\n",
    "            print(f\"- {dep}\")\n",
    "\n",
    "    # Dependents\n",
    "    dependents = [ds for ds, deps in full_dependencies.items() if dataset_name in deps]\n",
    "    print(f\"\\nTotal Dependents: {len(dependents)}\")\n",
    "    if dependents:\n",
    "        print(\"Dependents - other datasets that rely on the selected dataset:\")\n",
    "        for dep in dependents:\n",
    "            print(f\"- {dep}\")\n",
    "\n",
    "# Create the dropdown widget for selecting datasets\n",
    "dataset_dropdown = widgets.Dropdown(\n",
    "    options=sorted(list(full_dependencies.keys())),\n",
    "    description='Select a dataset:',\n",
    "    style={'description_width': 'initial'}  # Adjust the width of the description\n",
    ")\n",
    "\n",
    "# Create the interactive widget\n",
    "interact(display_dataset_dependencies, dataset_name=dataset_dropdown)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}