carlosmalt · carlosmalt · Feb 24, 2020 · Feb 25, 2020 · Feb 25, 2020 · Feb 26, 2020
diff --git a/experiments/CoDelTest/Untitled.ipynb b/experiments/CoDelTest/Untitled.ipynb
@@ -0,0 +1,79 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "1 log PriorityItem(priority=4, item='#0000')\n",
+      "2 log PriorityItem(priority=3, item='#0001')\n",
+      "3 log PriorityItem(priority=2, item='#0002')\n",
+      "3 repair PriorityItem(priority=3, item='#0001')\n",
+      "4 log PriorityItem(priority=1, item='#0003')\n",
+      "6 repair PriorityItem(priority=1, item='#0003')\n",
+      "9 repair PriorityItem(priority=2, item='#0002')\n",
+      "12 repair PriorityItem(priority=4, item='#0000')\n"
+     ]
+    }
+   ],
+   "source": [
+    "import simpy\n",
+    "import random\n",
+    "import math\n",
+    "\n",
+    "env = simpy.Environment()\n",
+    "issues = simpy.PriorityStore(env)\n",
+    "def inspector(env, issues):\n",
+    "    for issue in [simpy.PriorityItem(4, '#0000'),\n",
+    "                   simpy.PriorityItem(3, '#0001'),\n",
+    "                   simpy.PriorityItem(2, '#0002'),\n",
+    "                   simpy.PriorityItem(1, '#0003')]:\n",
+    "        yield env.timeout(1)\n",
+    "        print(env.now, 'log', issue)\n",
+    "        yield issues.put(issue)\n",
+    "\n",
+    "def maintainer(env, issues):\n",
+    "    while True:\n",
+    "        yield env.timeout(3)\n",
+    "        issue = yield issues.get()\n",
+    "        print(env.now, 'repair', issue)\n",
+    "\n",
+    "_ = env.process(inspector(env, issues))\n",
+    "_ = env.process(maintainer(env, issues))\n",
+    "env.run()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/experiments/CoDelTest/codel-rule01-res.ipynb b/experiments/CoDelTest/codel-rule01-res.ipynb
diff --git a/experiments/CoDelTest/codel-rule02-res.ipynb b/experiments/CoDelTest/codel-rule02-res.ipynb
diff --git a/experiments/CoDelTest/codel-rule03-res.ipynb b/experiments/CoDelTest/codel-rule03-res.ipynb
diff --git a/experiments/CoDelTest/codel-rule04-res.ipynb b/experiments/CoDelTest/codel-rule04-res.ipynb
diff --git a/experiments/CoDelTest/plot.py b/experiments/CoDelTest/plot.py
@@ -0,0 +1,32 @@
+import matplotlib.pyplot as plt
+from statistics import mean
+
+# get average bs_lat(bs_lat is the latency in bluestore)
+bs_lat_list = [x[1] for x in bs_lat_vec]
+total_lat_list = [x[1] for x in total_lat_vec]
+
+priInv = 0 # number of priority inversion
+txn_numbers = len(bs_lat_vec)
+for i in range(txn_numbers - 1):
+    if bs_lat_vec[i][0] < bs_lat_vec[i+1][0]:
+        priInv = priInv + 1
+print("number of priority inversion =",priInv, "number of txcs =", txn_numbers)
+
+fig, ax = plt.subplots()
+#ax.set_yscale('log')
+ax.plot(blocking_dur_vec, 'b-')
+ax.set(xlabel='time', ylabel='blocking_duration[us]', title='blocking duration over time')
+
+print("average bluestore latency(s) =",mean(bs_lat_list)/1000000)
+print("average total latency(s) =",mean(total_lat_list)/1000000)
+
+print("average osd queue size =",mean(osd_queue_size_vec))
+fig, ax = plt.subplots()
+ax.plot(osd_queue_size_vec, 'r-')
+ax.set_ylim(0,14)
+ax.set(xlabel='time', ylabel='osd_queue size', title='osd queue size')
+
+print("average kv queue size =",mean(kv_queue_size_vec))
+fig, ax = plt.subplots()
+ax.plot(kv_queue_size_vec, 'c-')
+ax.set(xlabel='time', ylabel='kv_queue size', title='bluestore kv queue size')
diff --git a/experiments/CoDelTest/simulation-codel-out.ipynb b/experiments/CoDelTest/simulation-codel-out.ipynb
diff --git a/experiments/CoDelTest/simulation-codel.ipynb b/experiments/CoDelTest/simulation-codel.ipynb
diff --git a/experiments/latModel/BlueStore/base-batch-1/dump-txc-lat-n-size.png b/experiments/latModel/BlueStore/base-batch-1/dump-txc-lat-n-size.png
diff --git a/experiments/latModel/BlueStore/base-batch-1/latency-model-for-bluestore.ipynb b/experiments/latModel/BlueStore/base-batch-1/latency-model-for-bluestore.ipynb
diff --git a/experiments/latModel/BlueStore/README.md → ...explore_bluestore_latency_model/README.md b/experiments/latModel/BlueStore/README.md → ...explore_bluestore_latency_model/README.md
@@ -1,9 +1,17 @@
 ## The experiment in Ceph with BlueStore
 
-`base-batch-1`:
+`first-exp-batch01`:
 1. CoDel is disabled
 2. committing batch in BlueStore is set to 1
 3. except `dump-codel-tests.csv`, all other data are raw data
 4. `dump-codel-tests.csv` is generated in BlueStore. The first and last 5 data points are removed for steady state. This file is used in the plot script. 
 5. The data is getting from my branch: https://github.com/yzhan298/ceph.git, research\_rs\_mg
 6. In order to dump these csvs, you need to call ceph admin socket commnad: `bin/ceph daemon osd.0 dump kvq vector` 
+This is the first experiment to get the latency model, so many of the files are not optimized for data process.
+
+The `batch01`, `batch02`, `batch05` are new experiments. All csv files are raw data.
+The cooking script is in the jupyter notebook.
+`batch01`: commiting batch size = 1
+`batch02`: commiting batch size = 2
+`batch05`: commiting batch size = 5
+
diff --git a/...xplore_bluestore_latency_model/batch01-rocksdb-memtable/analysis-memtable-increased.ipynb b/...xplore_bluestore_latency_model/batch01-rocksdb-memtable/analysis-memtable-increased.ipynb
diff --git a/...y_model/batch01-rocksdb-memtable/difference-between-kv_sync_lat-and-kvq_lat.png b/...y_model/batch01-rocksdb-memtable/difference-between-kv_sync_lat-and-kvq_lat.png