Merge pull request #98 from Sharon-iguazio/gpu-demos.pr97-updated-dirs-edit

RAPIDS GPU Demos: MERGE PR 97 "Updated dirs" to developoment + fix outputs count & rmv empty cell

Author: Sharon-iguazio

demos/gpu/rapids/benchmark-cudf-vs-pd.ipynb

@@ -295,46 +295,52 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Configurations"
+    "### Imports"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Benchmark configurations\n",
-    "metric_names = ['cpu_utilization', 'latency', 'packet_loss', 'throughput']\n",
-    "nlargest = 10"
+    "import os\n",
+    "import yaml\n",
+    "import time\n",
+    "import datetime\n",
+    "import json\n",
+    "import itertools\n",
+    "\n",
+    "# Generator\n",
+    "from v3io_generator import metrics_generator, deployment_generator\n",
+    "\n",
+    "# Dataframes\n",
+    "import cudf\n",
+    "import pandas as pd"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Imports"
+    "### Configurations"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
-    "import os\n",
-    "import yaml\n",
-    "import time\n",
-    "import datetime\n",
-    "import json\n",
-    "import itertools\n",
+    "# Benchmark configurations\n",
+    "metric_names = ['cpu_utilization', 'latency', 'packet_loss', 'throughput']\n",
+    "nlargest = 10\n",
+    "source_file = os.path.join(os.getcwd(), 'data', 'ops.logs') # Use full path\n",
     "\n",
-    "# Generator\n",
-    "from v3io_generator import metrics_generator, deployment_generator\n",
     "\n",
-    "# Dataframes\n",
-    "import cudf\n",
-    "import pandas as pd"
+    "os.environ['SOURCE_PATH'] = source_file                    # Expose for display\n",
+    "os.environ['SOURCE_DIR'] = os.path.dirname(source_file)    # Expose for display\n",
+    "os.environ['SOURCE_FILE'] = os.path.basename(source_file)  # Expose for display"
    ]
   },
   {
@@ -434,7 +440,7 @@
        "4                Carr-Reyes                0        0            0           0"
       ]
      },
-     "execution_count": 6,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -457,9 +463,16 @@
     "deployment_df.head()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Specify metrics configuration for the generator"
+   ]
+  },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -515,14 +528,10 @@
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": 8,
+   "cell_type": "markdown",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "met_gen = metrics_generator.Generator_df(metrics_configuration, \n",
-    "                                         user_hierarchy=deployment_df, \n",
-    "                                         initial_timestamp=time.time())"
+    "Create the data according to the given hierarchy and metrics configuration"
    ]
   },
   {
@@ -531,15 +540,21 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "source_file = '/tmp/ops.logs'\n",
+    "met_gen = metrics_generator.Generator_df(metrics_configuration, \n",
+    "                                         user_hierarchy=deployment_df, \n",
+    "                                         initial_timestamp=time.time())\n",
+    "\n",
     "metrics = met_gen.generate_range(start_time=datetime.datetime.now(),\n",
     "                                 end_time=datetime.datetime.now()+datetime.timedelta(hours=62),\n",
     "                                 as_df=True,\n",
     "                                 as_iterator=False)\n",
     "\n",
+    "# Verify source file directory exists\n",
+    "os.makedirs(os.path.dirname(source_file), exist_ok=1)\n",
+    "\n",
     "# Generate file from metrics\n",
     "with open(source_file, 'w') as f:\n",
-    "    metrics_batch = metrics\n",
+    "    metrics_batch = metric_names\n",
     "    metrics_batch.to_json(f,\n",
     "                          orient='records',\n",
     "                          lines=True)"
@@ -562,15 +577,12 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "total 0\n",
-      "drwxr-xr-x 2 50 nogroup    0 Jul 22 11:32 .ipynb_checkpoints\n",
-      "-rw-r--r-- 1 50 nogroup 1.2G Jul 22 07:17 ops-1gb.logs\n",
-      "-rw-r--r-- 1 50 nogroup 1.2G Jul 22 11:40 ops.logs\n"
+      "-rw-r--r-- 1 50 nogroup 1.2G Aug 13 11:00 ops.logs\n"
      ]
     }
    ],
    "source": [
-    "!ls -lah data"
+    "!ls -lah ${SOURCE_DIR} | grep ${SOURCE_FILE}"
    ]
   },
   {
@@ -582,21 +594,21 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "{\"company\":\"Rios__Pope_and_Baird\",\"cpu_utilization\":70.6942165035,\"cpu_utilization_is_error\":false,\"latency\":3.1373003261,\"latency_is_error\":false,\"packet_loss\":0.0,\"packet_loss_is_error\":false,\"throughput\":249.7207880994,\"throughput_is_error\":false,\"timestamp\":1563795193534}\n",
-      "{\"company\":\"Ross__Calderon_and_Brown\",\"cpu_utilization\":56.540474522,\"cpu_utilization_is_error\":false,\"latency\":0.0,\"latency_is_error\":false,\"packet_loss\":0.0,\"packet_loss_is_error\":false,\"throughput\":261.9362588938,\"throughput_is_error\":false,\"timestamp\":1563795193534}\n",
-      "{\"company\":\"Jackson_PLC\",\"cpu_utilization\":75.7476859549,\"cpu_utilization_is_error\":false,\"latency\":0.0,\"latency_is_error\":false,\"packet_loss\":1.3991427041,\"packet_loss_is_error\":false,\"throughput\":221.8819458316,\"throughput_is_error\":false,\"timestamp\":1563795193534}\n",
-      "{\"company\":\"Reyes_Group\",\"cpu_utilization\":61.4657850595,\"cpu_utilization_is_error\":false,\"latency\":0.0,\"latency_is_error\":false,\"packet_loss\":1.7039267608,\"packet_loss_is_error\":false,\"throughput\":232.4317426357,\"throughput_is_error\":false,\"timestamp\":1563795193534}\n",
-      "{\"company\":\"Carr-Reyes\",\"cpu_utilization\":49.9688296158,\"cpu_utilization_is_error\":false,\"latency\":0.4559851085,\"latency_is_error\":false,\"packet_loss\":0.0,\"packet_loss_is_error\":false,\"throughput\":276.0540712289,\"throughput_is_error\":false,\"timestamp\":1563795193534}\n",
-      "{\"company\":\"Wilson_and_Sons\",\"cpu_utilization\":66.0718786965,\"cpu_utilization_is_error\":false,\"latency\":0.0,\"latency_is_error\":false,\"packet_loss\":0.2971395649,\"packet_loss_is_error\":false,\"throughput\":264.7801359998,\"throughput_is_error\":false,\"timestamp\":1563795193534}\n",
-      "{\"company\":\"Nolan__Norton_and_Best\",\"cpu_utilization\":63.0572487273,\"cpu_utilization_is_error\":false,\"latency\":0.0,\"latency_is_error\":false,\"packet_loss\":0.0,\"packet_loss_is_error\":false,\"throughput\":264.033448601,\"throughput_is_error\":false,\"timestamp\":1563795193534}\n",
-      "{\"company\":\"Brandt_LLC\",\"cpu_utilization\":90.6382973539,\"cpu_utilization_is_error\":false,\"latency\":1.9411864095,\"latency_is_error\":false,\"packet_loss\":2.527867727,\"packet_loss_is_error\":false,\"throughput\":232.8742238776,\"throughput_is_error\":false,\"timestamp\":1563795193534}\n",
-      "{\"company\":\"Williams-Collins\",\"cpu_utilization\":64.1690341745,\"cpu_utilization_is_error\":false,\"latency\":0.0,\"latency_is_error\":false,\"packet_loss\":3.2983607178,\"packet_loss_is_error\":false,\"throughput\":264.0176356426,\"throughput_is_error\":false,\"timestamp\":1563795193534}\n",
-      "{\"company\":\"Williams__Hutchinson_and_Harrison\",\"cpu_utilization\":60.6966740664,\"cpu_utilization_is_error\":false,\"latency\":0.0,\"latency_is_error\":false,\"packet_loss\":3.1742385435,\"packet_loss_is_error\":false,\"throughput\":230.875976139,\"throughput_is_error\":false,\"timestamp\":1563795193534}\n"
+      "{\"company\":\"Parker_and_Sons\",\"cpu_utilization\":77.0001379709,\"cpu_utilization_is_error\":false,\"latency\":8.9685908315,\"latency_is_error\":false,\"packet_loss\":0.1182060132,\"packet_loss_is_error\":false,\"throughput\":264.5567821475,\"throughput_is_error\":false,\"timestamp\":1565015264829}\n",
+      "{\"company\":\"Barnes-Fletcher\",\"cpu_utilization\":82.3951249969,\"cpu_utilization_is_error\":false,\"latency\":2.7561547101,\"latency_is_error\":false,\"packet_loss\":0.9046704441,\"packet_loss_is_error\":false,\"throughput\":260.4376162919,\"throughput_is_error\":false,\"timestamp\":1565015264829}\n",
+      "{\"company\":\"Johnson_Ltd\",\"cpu_utilization\":74.2230353639,\"cpu_utilization_is_error\":false,\"latency\":7.0199027791,\"latency_is_error\":false,\"packet_loss\":0.0,\"packet_loss_is_error\":false,\"throughput\":245.2035029337,\"throughput_is_error\":false,\"timestamp\":1565015264829}\n",
+      "{\"company\":\"Cameron_Ltd\",\"cpu_utilization\":61.9061750617,\"cpu_utilization_is_error\":false,\"latency\":6.8589515103,\"latency_is_error\":false,\"packet_loss\":0.0,\"packet_loss_is_error\":false,\"throughput\":245.3090546657,\"throughput_is_error\":false,\"timestamp\":1565015264829}\n",
+      "{\"company\":\"Sims-Mccoy\",\"cpu_utilization\":69.8156710076,\"cpu_utilization_is_error\":false,\"latency\":0.0,\"latency_is_error\":false,\"packet_loss\":4.1570694474,\"packet_loss_is_error\":false,\"throughput\":238.5772519428,\"throughput_is_error\":false,\"timestamp\":1565015264829}\n",
+      "{\"company\":\"Bautista__Olson_and_Guzman\",\"cpu_utilization\":61.0139995818,\"cpu_utilization_is_error\":false,\"latency\":0.0,\"latency_is_error\":false,\"packet_loss\":0.0,\"packet_loss_is_error\":false,\"throughput\":253.2949507053,\"throughput_is_error\":false,\"timestamp\":1565015264829}\n",
+      "{\"company\":\"Anderson_Inc\",\"cpu_utilization\":72.6603863726,\"cpu_utilization_is_error\":false,\"latency\":3.6437115053,\"latency_is_error\":false,\"packet_loss\":0.0,\"packet_loss_is_error\":false,\"throughput\":267.6726148695,\"throughput_is_error\":false,\"timestamp\":1565015264829}\n",
+      "{\"company\":\"Nguyen-Silva\",\"cpu_utilization\":59.5127477315,\"cpu_utilization_is_error\":false,\"latency\":0.0,\"latency_is_error\":false,\"packet_loss\":0.0,\"packet_loss_is_error\":false,\"throughput\":255.2915162076,\"throughput_is_error\":false,\"timestamp\":1565015264829}\n",
+      "{\"company\":\"Patrick_Group\",\"cpu_utilization\":68.3079877253,\"cpu_utilization_is_error\":false,\"latency\":0.0,\"latency_is_error\":false,\"packet_loss\":2.6670360416,\"packet_loss_is_error\":false,\"throughput\":243.9449272531,\"throughput_is_error\":false,\"timestamp\":1565015264829}\n",
+      "{\"company\":\"Gutierrez_Ltd\",\"cpu_utilization\":70.1385482657,\"cpu_utilization_is_error\":false,\"latency\":0.0,\"latency_is_error\":false,\"packet_loss\":0.8048798451,\"packet_loss_is_error\":false,\"throughput\":252.1408356,\"throughput_is_error\":false,\"timestamp\":1565015264829}\n"
      ]
     }
    ],
    "source": [
-    "!head data/ops.logs"
+    "!head ${SOURCE_PATH}"
    ]
   },
   {
@@ -613,7 +625,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -629,14 +641,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "1.44 s ± 23.1 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
+      "6.24 s ± 62.3 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
      ]
     }
    ],
@@ -663,14 +675,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "43.4 s ± 627 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
+      "47.6 s ± 228 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
      ]
     }
    ],
@@ -704,14 +716,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 35,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "1.2 s ± 120 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
+      "5.95 s ± 77.3 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
      ]
     }
    ],
@@ -729,7 +741,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 36,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [
     {
@@ -755,7 +767,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -772,7 +784,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 18,
    "metadata": {},
    "outputs": [
     {
@@ -800,7 +812,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 19,
    "metadata": {},
    "outputs": [
     {
@@ -818,13 +830,6 @@
     "            agg({k: ['min', 'max', 'mean'] for k in metric_names})\n",
     "raw_nlargest = pdf.nlargest(nlargest, 'cpu_utilization')"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {
@@ -847,5 +852,5 @@
   }
  },
  "nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
 }