notebooks/create-validation-dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "51a2b01a-6ed3-40ba-91e1-9ce499d11a07",
   "metadata": {},
   "source": [
    "## Validation dataset\n",
    "This notebook takes the FAQ questionnaire from the ROSA workshop documents and creates a fine-tuning or validation dataset for text generation models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "42bf2292-252e-4bd7-8fe3-a93ba7134a0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "pd.set_option('display.max_colwidth', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "73e7542a-45a2-4ce7-aa74-a8a5317e08af",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "### Generated using successive iterations of Chat GPT code (minor edits) \n",
    "\n",
    "with open('../data/external/rosaworkshop/14-faq.md', 'r') as f:\n",
    "    contents = f.read()\n",
    "\n",
    "# Split the text into lines\n",
    "lines = contents.split('\\n')\n",
    "\n",
    "# Initialize variables to store the current question and its answer\n",
    "current_question = None\n",
    "current_answer = None\n",
    "\n",
    "question_answer = {}\n",
    "\n",
    "# Loop through each line of the text\n",
    "for line in lines:\n",
    "    # Check if the line starts with '###', indicating a new question\n",
    "    if line.startswith('###'):\n",
    "        # If there was a previous question, print it and its answer\n",
    "        if current_question and current_answer:\n",
    "            question_answer[current_question] = current_answer\n",
    "        # Set the new question as the current question\n",
    "        current_question = line[4:]\n",
    "        # Reset the current answer\n",
    "        current_answer = None\n",
    "    if line.startswith('#'):\n",
    "        continue\n",
    "    # Otherwise, the line is an answer to the current question\n",
    "    elif line != '':\n",
    "        # Append the answer to the current answer (if any)\n",
    "        if current_answer:\n",
    "            current_answer += '\\n' + line\n",
    "        else:\n",
    "            current_answer = line\n",
    "\n",
    "# Print the last question and its answer (if any)\n",
    "if current_question and current_answer:\n",
    "    question_answer[current_question] = current_answer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9e3bd68b-823d-4333-9985-e423c7fb2a0d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What is Red Hat OpenShift Service on AWS (ROSA)?</td>\n",
       "      <td>Red Hat Openshift Service on AWS (ROSA) is a fully-managed turnkey application platform that allows you to focus on what matters most, delivering value to your customers by building and deploying applications. Red Hat SRE experts manage the underlying platform so you don’t have to worry about the complexity of infrastructure management.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Where can I go to get more information/details?</td>\n",
       "      <td>- [ROSA Webpage](https://www.openshift.com/products/amazon-openshift)\\n- [ROSA Workshop](https://www.rosaworkshop.io)\\n- [ROSA Documentation](https://docs.openshift.com/rosa/welcome/index.html)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>What are the benefits of Red Hat OpenShift Service on AWS (Key Features)?</td>\n",
       "      <td>- **Native AWS service:** Access and use Red Hat OpenShift on demand with a self-service on-boarding experience through the AWS management console.\\n- **Flexible, consumption-based pricing:** Scale as per your business needs and pay as you go with flexible pricing with an on-demand hourly or annual billing model.\\n- **Single bill for Red Hat OpenShift &amp; AWS usage:**  Customers will receive a single bill from AWS for both Red Hat OpenShift and AWS consumption.\\n- **Fully integrated support experience:** Installation, management, maintenance, and upgrades are performed by Red Hat site reliability engineers (SRE) with joint Red Hat and Amazon support and a 99.95% SLA.\\n- **AWS service integration:** AWS has a robust portfolio of cloud services, such as compute, storage, networking, database, analytics, and machine learning, which are directly accessible via Red Hat OpenShift Service on AWS. This makes it easier to build, operate, and scale globally on demand through a familiar management interface.\\nAdditional key features of Red Hat OpenShift Service on AWS:\\n- **Maximum Availability:** Deploy clusters across multiple Availability Zones in supported Regions to maximize availability to maintain high availability for your most demanding mission-critical applications and data.\\n- **Cluster node scaling:** Easily add or remove compute nodes to match resource demand\\n- **Optimized clusters:** Choose from memory-optimized, compute-optimized, or general purpose EC2 instance types, with clusters sized to meet your needs. See [AWS compute types](https://docs.openshift.com/rosa/rosa_architecture/rosa_policy_service_definition/rosa-service-definition.html#rosa-sdpolicy-aws-compute-types_rosa-service-definition).\\n- **Global availability:** Please refer to the [product regional availability page](https://docs.openshift.com/rosa/rosa_architecture/rosa_policy_service_definition/rosa-service-definition.html#rosa-sdpolicy-regions-az_rosa-service-definition) page for an up-to-date view of where Red Hat OpenShift Service on AWS is available.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What are the differences between Red Hat OpenShift Service on AWS and Kubernetes?</td>\n",
       "      <td>Everything you need to deploy and manage containers is bundled with ROSA, including container management, automation (Operators), networking, load balancing, service mesh, CI/CD, firewall, monitoring, registry, authentication, and authorization capabilities. These components are tested together for unified operations as a complete platform. Automated cluster operations, including over-the-air platform upgrades, further enhance your Kubernetes experience.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>What exactly am I responsible for and what is Red Hat / AWS responsible for?</td>\n",
       "      <td>In short, anything that is related to deploying the cluster or keeping the cluster running will be Red Hat’s or AWS’s responsibility, and anything relating to the applications, users, or data is the customers responsibility.  Please see our [responsibility matrix](https://docs.openshift.com/rosa/rosa_architecture/rosa_policy_service_definition/rosa-policy-responsibility-matrix.html) for more details.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>What features are upcoming for ROSA?</td>\n",
       "      <td>The current ROSA roadmap can be seen at: [https://red.ht/rosa-roadmap](https://red.ht/rosa-roadmap)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>What kind of instances are supported for worker nodes?</td>\n",
       "      <td>See [AWS compute types](https://docs.openshift.com/rosa/rosa_architecture/rosa_policy_service_definition/rosa-service-definition.html#rosa-sdpolicy-aws-instance-types_rosa-service-definition) in the service definition for the up to date list of supported instance types.  Additionally, spot instances are supported.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>Does ROSA support an air-gapped, disconnected environment where the ROSA cluster does not have internet access?</td>\n",
       "      <td>No, the ROSA cluster must have egress to the internet to access our registry, S3, send metrics etc. The service requires a number of [egress endpoints](https://docs.openshift.com/rosa/rosa_install_access_delete_clusters/rosa_getting_started_iam/rosa-aws-prereqs.html#osd-aws-privatelink-firewall-prerequisites). Ingress can be limited to PrivateLink (for Red Hat SRE) and VPN or similar for customer access.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>Is node autoscaling available?</td>\n",
       "      <td>Yes. Autoscaling allows you to automatically adjust the size of the cluster based on the current workload. See [About autoscaling nodes on a cluster](https://docs.openshift.com/rosa/rosa_cluster_admin/rosa_nodes/rosa-nodes-about-autoscaling-nodes.html) in the documentation for more details.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>What is the maximum number of worker nodes that a cluster can support?</td>\n",
       "      <td>The maximum number of worker nodes is 180 per ROSA cluster.  See here for [limits and scalability](https://docs.openshift.com/rosa/rosa_planning/rosa-limits-scalability.html) considerations and more details on node counts.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>65 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                             Question  \\\n",
       "0                                                                    What is Red Hat OpenShift Service on AWS (ROSA)?   \n",
       "1                                                                     Where can I go to get more information/details?   \n",
       "2                                           What are the benefits of Red Hat OpenShift Service on AWS (Key Features)?   \n",
       "3                                   What are the differences between Red Hat OpenShift Service on AWS and Kubernetes?   \n",
       "4                                        What exactly am I responsible for and what is Red Hat / AWS responsible for?   \n",
       "..                                                                                                                ...   \n",
       "60                                                                               What features are upcoming for ROSA?   \n",
       "61                                                             What kind of instances are supported for worker nodes?   \n",
       "62  Does ROSA support an air-gapped, disconnected environment where the ROSA cluster does not have internet access?     \n",
       "63                                                                                     Is node autoscaling available?   \n",
       "64                                             What is the maximum number of worker nodes that a cluster can support?   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       Answer  \n",
       "0                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          Red Hat Openshift Service on AWS (ROSA) is a fully-managed turnkey application platform that allows you to focus on what matters most, delivering value to your customers by building and deploying applications. Red Hat SRE experts manage the underlying platform so you don’t have to worry about the complexity of infrastructure management.  \n",
       "1                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           - [ROSA Webpage](https://www.openshift.com/products/amazon-openshift)\\n- [ROSA Workshop](https://www.rosaworkshop.io)\\n- [ROSA Documentation](https://docs.openshift.com/rosa/welcome/index.html)  \n",
       "2   - **Native AWS service:** Access and use Red Hat OpenShift on demand with a self-service on-boarding experience through the AWS management console.\\n- **Flexible, consumption-based pricing:** Scale as per your business needs and pay as you go with flexible pricing with an on-demand hourly or annual billing model.\\n- **Single bill for Red Hat OpenShift & AWS usage:**  Customers will receive a single bill from AWS for both Red Hat OpenShift and AWS consumption.\\n- **Fully integrated support experience:** Installation, management, maintenance, and upgrades are performed by Red Hat site reliability engineers (SRE) with joint Red Hat and Amazon support and a 99.95% SLA.\\n- **AWS service integration:** AWS has a robust portfolio of cloud services, such as compute, storage, networking, database, analytics, and machine learning, which are directly accessible via Red Hat OpenShift Service on AWS. This makes it easier to build, operate, and scale globally on demand through a familiar management interface.\\nAdditional key features of Red Hat OpenShift Service on AWS:\\n- **Maximum Availability:** Deploy clusters across multiple Availability Zones in supported Regions to maximize availability to maintain high availability for your most demanding mission-critical applications and data.\\n- **Cluster node scaling:** Easily add or remove compute nodes to match resource demand\\n- **Optimized clusters:** Choose from memory-optimized, compute-optimized, or general purpose EC2 instance types, with clusters sized to meet your needs. See [AWS compute types](https://docs.openshift.com/rosa/rosa_architecture/rosa_policy_service_definition/rosa-service-definition.html#rosa-sdpolicy-aws-compute-types_rosa-service-definition).\\n- **Global availability:** Please refer to the [product regional availability page](https://docs.openshift.com/rosa/rosa_architecture/rosa_policy_service_definition/rosa-service-definition.html#rosa-sdpolicy-regions-az_rosa-service-definition) page for an up-to-date view of where Red Hat OpenShift Service on AWS is available.  \n",
       "3                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  Everything you need to deploy and manage containers is bundled with ROSA, including container management, automation (Operators), networking, load balancing, service mesh, CI/CD, firewall, monitoring, registry, authentication, and authorization capabilities. These components are tested together for unified operations as a complete platform. Automated cluster operations, including over-the-air platform upgrades, further enhance your Kubernetes experience.  \n",
       "4                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         In short, anything that is related to deploying the cluster or keeping the cluster running will be Red Hat’s or AWS’s responsibility, and anything relating to the applications, users, or data is the customers responsibility.  Please see our [responsibility matrix](https://docs.openshift.com/rosa/rosa_architecture/rosa_policy_service_definition/rosa-policy-responsibility-matrix.html) for more details.  \n",
       "..                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        ...  \n",
       "60                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        The current ROSA roadmap can be seen at: [https://red.ht/rosa-roadmap](https://red.ht/rosa-roadmap)  \n",
       "61                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                See [AWS compute types](https://docs.openshift.com/rosa/rosa_architecture/rosa_policy_service_definition/rosa-service-definition.html#rosa-sdpolicy-aws-instance-types_rosa-service-definition) in the service definition for the up to date list of supported instance types.  Additionally, spot instances are supported.  \n",
       "62                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    No, the ROSA cluster must have egress to the internet to access our registry, S3, send metrics etc. The service requires a number of [egress endpoints](https://docs.openshift.com/rosa/rosa_install_access_delete_clusters/rosa_getting_started_iam/rosa-aws-prereqs.html#osd-aws-privatelink-firewall-prerequisites). Ingress can be limited to PrivateLink (for Red Hat SRE) and VPN or similar for customer access.  \n",
       "63                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        Yes. Autoscaling allows you to automatically adjust the size of the cluster based on the current workload. See [About autoscaling nodes on a cluster](https://docs.openshift.com/rosa/rosa_cluster_admin/rosa_nodes/rosa-nodes-about-autoscaling-nodes.html) in the documentation for more details.  \n",
       "64                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             The maximum number of worker nodes is 180 per ROSA cluster.  See here for [limits and scalability](https://docs.openshift.com/rosa/rosa_planning/rosa-limits-scalability.html) considerations and more details on node counts.  \n",
       "\n",
       "[65 rows x 2 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validation_set = pd.DataFrame(question_answer.items())\n",
    "validation_set.rename(columns={0:'Question', 1:'Answer'}, inplace=True)\n",
    "validation_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2dc72548-849e-4dd1-ae45-d66d3a197708",
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_set.to_csv('../data/processed/validation_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7fda387-277c-49a8-9a6b-2658cfce2be1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}