Skip to content

Commit 1813b5d

Browse files
gilad-shahamgilad-shaham
authored
Merge pull request #336 from v3io/development
fix dask-cluster.ipynb and link to mlrun doc site
2 parents 4efb21e + 8ee5f7e commit 1813b5d

File tree

3 files changed

+57
-55
lines changed

3 files changed

+57
-55
lines changed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ The Iguazio Data Science Platform (**"the platform"**) is a fully integrated and
2020
The platform incorporates the following components:
2121

2222
- A data science workbench that includes Jupyter Notebook, integrated analytics engines, and Python packages
23-
- The [MLRun](https://github.com/mlrun/mlrun) open-source MLOps orchestration framework for ML model management with experiments tracking and pipeline automation
23+
- The [MLRun](https://mlrun.readthedocs.io) open-source MLOps orchestration framework for ML model management with experiments tracking and pipeline automation
2424
- Managed data and machine-learning (ML) services over a scalable Kubernetes cluster
2525
- A real-time serverless functions framework for model serving ([Nuclio](https://nuclio.io/))
2626
- An extremely fast and secure data layer that supports SQL, NoSQL, time-series databases, files (simple objects), and streaming

‎data-ingestion-and-preparation/dask-cluster.ipynb‎

Lines changed: 55 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,17 @@
3434
},
3535
{
3636
"cell_type": "code",
37-
"execution_count": null,
37+
"execution_count": 1,
3838
"metadata": {},
39-
"outputs": [],
39+
"outputs": [
40+
{
41+
"name": "stdout",
42+
"output_type": "stream",
43+
"text": [
44+
"Both server & client are aligned (0.6.0rc13).\n"
45+
]
46+
}
47+
],
4048
"source": [
4149
"#Make sure thar mlrun is installed. if it's already installed then skip this step\n",
4250
"#to instlal mlrun run the following\n",
@@ -53,7 +61,7 @@
5361
},
5462
{
5563
"cell_type": "code",
56-
"execution_count": 1,
64+
"execution_count": 2,
5765
"metadata": {},
5866
"outputs": [],
5967
"source": [
@@ -62,7 +70,7 @@
6270
},
6371
{
6472
"cell_type": "code",
65-
"execution_count": 2,
73+
"execution_count": 3,
6674
"metadata": {},
6775
"outputs": [
6876
{
@@ -71,7 +79,7 @@
7179
"text": [
7280
" % Total % Received % Xferd Average Speed Time Time Time Current\n",
7381
" Dload Upload Total Spent Left Speed\n",
74-
"100 84.9M 100 84.9M 0 0 21.8M 0 0:00:03 0:00:03 --:--:-- 21.8M\n"
82+
"100 84.9M 100 84.9M 0 0 8206k 0 0:00:10 0:00:10 --:--:-- 8639k\n"
7583
]
7684
}
7785
],
@@ -83,7 +91,7 @@
8391
},
8492
{
8593
"cell_type": "code",
86-
"execution_count": 3,
94+
"execution_count": 4,
8795
"metadata": {},
8896
"outputs": [],
8997
"source": [
@@ -93,7 +101,7 @@
93101
},
94102
{
95103
"cell_type": "code",
96-
"execution_count": 4,
104+
"execution_count": 5,
97105
"metadata": {},
98106
"outputs": [
99107
{
@@ -113,7 +121,7 @@
113121
},
114122
{
115123
"cell_type": "code",
116-
"execution_count": 5,
124+
"execution_count": 6,
117125
"metadata": {},
118126
"outputs": [],
119127
"source": [
@@ -149,7 +157,7 @@
149157
},
150158
{
151159
"cell_type": "code",
152-
"execution_count": 6,
160+
"execution_count": 7,
153161
"metadata": {},
154162
"outputs": [],
155163
"source": [
@@ -170,7 +178,7 @@
170178
},
171179
{
172180
"cell_type": "code",
173-
"execution_count": 7,
181+
"execution_count": 8,
174182
"metadata": {},
175183
"outputs": [],
176184
"source": [
@@ -186,7 +194,7 @@
186194
},
187195
{
188196
"cell_type": "code",
189-
"execution_count": 8,
197+
"execution_count": 9,
190198
"metadata": {},
191199
"outputs": [],
192200
"source": [
@@ -212,7 +220,7 @@
212220
},
213221
{
214222
"cell_type": "code",
215-
"execution_count": 9,
223+
"execution_count": 10,
216224
"metadata": {},
217225
"outputs": [],
218226
"source": [
@@ -228,14 +236,14 @@
228236
},
229237
{
230238
"cell_type": "code",
231-
"execution_count": 10,
239+
"execution_count": 11,
232240
"metadata": {},
233241
"outputs": [
234242
{
235243
"name": "stdout",
236244
"output_type": "stream",
237245
"text": [
238-
"> 2020-12-20 09:58:46,894 [info] using in-cluster config.\n"
246+
"> 2021-02-04 19:07:34,933 [info] using in-cluster config.\n"
239247
]
240248
}
241249
],
@@ -245,7 +253,7 @@
245253
},
246254
{
247255
"cell_type": "code",
248-
"execution_count": 11,
256+
"execution_count": 12,
249257
"metadata": {},
250258
"outputs": [],
251259
"source": [
@@ -258,21 +266,21 @@
258266
},
259267
{
260268
"cell_type": "code",
261-
"execution_count": 12,
269+
"execution_count": 13,
262270
"metadata": {},
263271
"outputs": [
264272
{
265273
"name": "stdout",
266274
"output_type": "stream",
267275
"text": [
268-
"> 2020-12-20 09:58:46,980 [info] trying dask client at: tcp://mlrun-dask-init-9d8122b2-b.default-tenant:8786\n",
269-
"> 2020-12-20 09:58:47,000 [info] using remote dask scheduler (mlrun-dask-init-9d8122b2-b) at: tcp://mlrun-dask-init-9d8122b2-b.default-tenant:8786\n"
276+
"> 2021-02-04 19:07:35,158 [info] trying dask client at: tcp://mlrun-dask-init-b7725876-7.default-tenant:8786\n",
277+
"> 2021-02-04 19:07:35,234 [info] using remote dask scheduler (mlrun-dask-init-b7725876-7) at: tcp://mlrun-dask-init-b7725876-7.default-tenant:8786\n"
270278
]
271279
},
272280
{
273281
"data": {
274282
"text/html": [
275-
"<a href=\"http://default-tenant.app.product-new.iguazio-cd2.com:31433/status\" target=\"_blank\" >dashboard link: default-tenant.app.product-new.iguazio-cd2.com:31433</a>"
283+
"<a href=\"http://default-tenant.app.bsmhzmwkycjf.iguazio-cd2.com:30053/status\" target=\"_blank\" >dashboard link: default-tenant.app.bsmhzmwkycjf.iguazio-cd2.com:30053</a>"
276284
],
277285
"text/plain": [
278286
"<IPython.core.display.HTML object>"
@@ -289,32 +297,33 @@
289297
"<td style=\"vertical-align: top; border: 0px solid white\">\n",
290298
"<h3 style=\"text-align: left;\">Client</h3>\n",
291299
"<ul style=\"text-align: left; list-style: none; margin: 0; padding: 0;\">\n",
292-
" <li><b>Scheduler: </b>tcp://mlrun-dask-init-9d8122b2-b.default-tenant:8786</li>\n",
293-
" <li><b>Dashboard: </b><a href='http://mlrun-dask-init-9d8122b2-b.default-tenant:8787/status' target='_blank'>http://mlrun-dask-init-9d8122b2-b.default-tenant:8787/status</a></li>\n",
300+
" <li><b>Scheduler: </b>tcp://mlrun-dask-init-b7725876-7.default-tenant:8786</li>\n",
301+
" <li><b>Dashboard: </b><a href='http://mlrun-dask-init-b7725876-7.default-tenant:8787/status' target='_blank'>http://mlrun-dask-init-b7725876-7.default-tenant:8787/status</a></li>\n",
294302
"</ul>\n",
295303
"</td>\n",
296304
"<td style=\"vertical-align: top; border: 0px solid white\">\n",
297305
"<h3 style=\"text-align: left;\">Cluster</h3>\n",
298306
"<ul style=\"text-align: left; list-style:none; margin: 0; padding: 0;\">\n",
299-
" <li><b>Workers: </b>1</li>\n",
300-
" <li><b>Cores: </b>1</li>\n",
301-
" <li><b>Memory: </b>4.15 GB</li>\n",
307+
" <li><b>Workers: </b>2</li>\n",
308+
" <li><b>Cores: </b>2</li>\n",
309+
" <li><b>Memory: </b>8.31 GB</li>\n",
302310
"</ul>\n",
303311
"</td>\n",
304312
"</tr>\n",
305313
"</table>"
306314
],
307315
"text/plain": [
308-
"<Client: 'tcp://10.200.0.53:8786' processes=1 threads=1, memory=4.15 GB>"
316+
"<Client: 'tcp://10.200.0.59:8786' processes=2 threads=2, memory=8.31 GB>"
309317
]
310318
},
311-
"execution_count": 12,
319+
"execution_count": 13,
312320
"metadata": {},
313321
"output_type": "execute_result"
314322
}
315323
],
316324
"source": [
317-
"dsf.client"
325+
"client = dsf.client\n",
326+
"client"
318327
]
319328
},
320329
{
@@ -326,12 +335,12 @@
326335
},
327336
{
328337
"cell_type": "code",
329-
"execution_count": 13,
338+
"execution_count": 14,
330339
"metadata": {},
331340
"outputs": [],
332341
"source": [
333342
"DATA_URL = '/User/examples/ytrip.csv'\n",
334-
"DASK_CLIENT = '<scheduler address - see above>'\n",
343+
"DASK_CLIENT = client.scheduler.address\n",
335344
"# e.g. DASK_CLIENT = 'tcp://mlrun-dask-init-9d8122b2-b.default-tenant:8786'"
336345
]
337346
},
@@ -351,16 +360,16 @@
351360
},
352361
{
353362
"cell_type": "code",
354-
"execution_count": 14,
363+
"execution_count": 15,
355364
"metadata": {},
356365
"outputs": [
357366
{
358367
"name": "stdout",
359368
"output_type": "stream",
360369
"text": [
361-
"> 2020-12-20 09:59:03,605 [info] starting run test-dask-test_dask uid=7e86bbe9112a486b84e9aa586638fd66 DB=http://mlrun-api:8080\n",
362-
"> 2020-12-20 09:59:03,761 [info] Job is running in the background, pod: test-dask-test-dask-8ws4f\n",
363-
"> 2020-12-20 09:59:20,372 [info] run executed, status=completed\n",
370+
"> 2021-02-04 19:07:35,288 [info] starting run test-dask-test_dask uid=4973d57c968747e491d6fcaa401968e0 DB=http://mlrun-api:8080\n",
371+
"> 2021-02-04 19:07:35,769 [info] Job is running in the background, pod: test-dask-test-dask-4m29c\n",
372+
"> 2021-02-04 19:07:56,270 [info] run executed, status=completed\n",
364373
"final state: completed\n"
365374
]
366375
},
@@ -535,26 +544,26 @@
535544
" <tbody>\n",
536545
" <tr>\n",
537546
" <td>default</td>\n",
538-
" <td><div title=\"7e86bbe9112a486b84e9aa586638fd66\"><a href=\"https://mlrun-ui.default-tenant.app.product-new.iguazio-cd2.com/projects/default/jobs/monitor/7e86bbe9112a486b84e9aa586638fd66/info\" target=\"_blank\" >...6638fd66</a></div></td>\n",
547+
" <td><div title=\"4973d57c968747e491d6fcaa401968e0\"><a href=\"https://dashboard.default-tenant.app.bsmhzmwkycjf.iguazio-cd2.com/mlprojects/default/jobs/monitor/4973d57c968747e491d6fcaa401968e0/info\" target=\"_blank\" >...401968e0</a></div></td>\n",
539548
" <td>0</td>\n",
540-
" <td>Dec 20 09:59:09</td>\n",
549+
" <td>Feb 04 19:07:42</td>\n",
541550
" <td>completed</td>\n",
542551
" <td>test-dask-test_dask</td>\n",
543-
" <td><div class=\"dictlist\">v3io_user=admin</div><div class=\"dictlist\">kind=job</div><div class=\"dictlist\">owner=admin</div><div class=\"dictlist\">host=test-dask-test-dask-8ws4f</div></td>\n",
544-
" <td><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"resultf521b185\" title=\"/files/examples/ytrip.csv\">dataset</div></td>\n",
545-
" <td><div class=\"dictlist\">dask_client=tcp://mlrun-dask-init-9d8122b2-b.default-tenant:8786</div></td>\n",
552+
" <td><div class=\"dictlist\">v3io_user=admin</div><div class=\"dictlist\">kind=job</div><div class=\"dictlist\">owner=admin</div><div class=\"dictlist\">host=test-dask-test-dask-4m29c</div></td>\n",
553+
" <td><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"resultf45735f7\" title=\"/files/examples/ytrip.csv\">dataset</div></td>\n",
554+
" <td><div class=\"dictlist\">dask_client=tcp://mlrun-dask-init-b7725876-7.default-tenant:8786</div></td>\n",
546555
" <td></td>\n",
547556
" <td></td>\n",
548557
" </tr>\n",
549558
" </tbody>\n",
550559
"</table>\n",
551560
"</div></div>\n",
552-
" <div id=\"resultf521b185-pane\" class=\"right-pane block hidden\">\n",
561+
" <div id=\"resultf45735f7-pane\" class=\"right-pane block hidden\">\n",
553562
" <div class=\"pane-header\">\n",
554-
" <span id=\"resultf521b185-title\" class=\"pane-header-title\">Title</span>\n",
555-
" <span onclick=\"closePanel(this)\" paneName=\"resultf521b185\" class=\"close clickable\">&times;</span>\n",
563+
" <span id=\"resultf45735f7-title\" class=\"pane-header-title\">Title</span>\n",
564+
" <span onclick=\"closePanel(this)\" paneName=\"resultf45735f7\" class=\"close clickable\">&times;</span>\n",
556565
" </div>\n",
557-
" <iframe class=\"fileview\" id=\"resultf521b185-body\"></iframe>\n",
566+
" <iframe class=\"fileview\" id=\"resultf45735f7-body\"></iframe>\n",
558567
" </div>\n",
559568
"</div>\n"
560569
],
@@ -570,17 +579,17 @@
570579
"output_type": "stream",
571580
"text": [
572581
"to track results use .show() or .logs() or in CLI: \n",
573-
"!mlrun get run 7e86bbe9112a486b84e9aa586638fd66 --project default , !mlrun logs 7e86bbe9112a486b84e9aa586638fd66 --project default\n",
574-
"> 2020-12-20 09:59:22,969 [info] run executed, status=completed\n"
582+
"!mlrun get run 4973d57c968747e491d6fcaa401968e0 --project default , !mlrun logs 4973d57c968747e491d6fcaa401968e0 --project default\n",
583+
"> 2021-02-04 19:08:05,372 [info] run executed, status=completed\n"
575584
]
576585
},
577586
{
578587
"data": {
579588
"text/plain": [
580-
"<mlrun.model.RunObject at 0x7fb52a0e8550>"
589+
"<mlrun.model.RunObject at 0x7efe58c67c50>"
581590
]
582591
},
583-
"execution_count": 14,
592+
"execution_count": 15,
584593
"metadata": {},
585594
"output_type": "execute_result"
586595
}
@@ -605,13 +614,6 @@
605614
"Users can view the progress and detailed information in the mlrun UI by clicking on the uid above. <br>\n",
606615
"Also, to track the dask progress in the dask UI click on the \"dashboard link\" above the \"client\" section"
607616
]
608-
},
609-
{
610-
"cell_type": "code",
611-
"execution_count": null,
612-
"metadata": {},
613-
"outputs": [],
614-
"source": []
615617
}
616618
],
617619
"metadata": {

‎welcome.ipynb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@
4141
"The platform incorporates the following components:\n",
4242
"\n",
4343
"- A data science workbench that includes Jupyter Notebook, integrated analytics engines, and Python packages\n",
44-
"- The [MLRun](https://github.com/mlrun/mlrun) open-source MLOps orchestration framework for ML model management with experiments tracking and pipeline automation\n",
44+
"- The [MLRun](https://mlrun.readthedocs.io) open-source MLOps orchestration framework for ML model management with experiments tracking and pipeline automation\n",
4545
"- Managed data and machine-learning (ML) services over a scalable Kubernetes cluster\n",
4646
"- A real-time serverless functions framework for model serving ([Nuclio](https://nuclio.io/))\n",
4747
"- An extremely fast and secure data layer that supports SQL, NoSQL, time-series databases, files (simple objects), and streaming\n",

0 commit comments

Comments
 (0)