Skip to content

Commit 45c6ce8

Browse files
Sharon-iguazioSharon-iguazio
committed
Merge branch 'development' of github.com:v3io/tutorials
2 parents 349247c + 8f0a04f commit 45c6ce8

File tree

11 files changed

+211
-161
lines changed

11 files changed

+211
-161
lines changed

demos/gpu/horovod-benchmark/running_horovod_benchmark.ipynb renamed to demos/gpu/horovod/benchmark-tf.ipynb

Lines changed: 5 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
},
1212
{
1313
"cell_type": "code",
14-
"execution_count": 10,
14+
"execution_count": 1,
1515
"metadata": {},
1616
"outputs": [],
1717
"source": [
@@ -20,7 +20,7 @@
2020
},
2121
{
2222
"cell_type": "code",
23-
"execution_count": 11,
23+
"execution_count": 2,
2424
"metadata": {},
2525
"outputs": [
2626
{
@@ -74,7 +74,7 @@
7474
},
7575
{
7676
"cell_type": "code",
77-
"execution_count": 14,
77+
"execution_count": 3,
7878
"metadata": {},
7979
"outputs": [
8080
{
@@ -98,7 +98,7 @@
9898
},
9999
{
100100
"cell_type": "code",
101-
"execution_count": 6,
101+
"execution_count": 4,
102102
"metadata": {},
103103
"outputs": [
104104
{
@@ -166,7 +166,7 @@
166166
},
167167
{
168168
"cell_type": "code",
169-
"execution_count": 7,
169+
"execution_count": 5,
170170
"metadata": {},
171171
"outputs": [
172172
{
@@ -187,13 +187,6 @@
187187
"source": [
188188
"job.delete()"
189189
]
190-
},
191-
{
192-
"cell_type": "code",
193-
"execution_count": null,
194-
"metadata": {},
195-
"outputs": [],
196-
"source": []
197190
}
198191
],
199192
"metadata": {

demos/gpu/horovod-image-classification/01-Load_data_cats_n_dogs.ipynb renamed to demos/gpu/horovod/image-classification/01-load-data-cats-n-dogs.ipynb

Lines changed: 22 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
"cell_type": "markdown",
55
"metadata": {},
66
"source": [
7-
"# Load cats & dogs images"
7+
"# Load Cats and Dogs Images"
88
]
99
},
1010
{
@@ -13,12 +13,12 @@
1313
"_uuid": "fe76d1d1ded592430e7548feacfa38dc42f085d9"
1414
},
1515
"source": [
16-
"# Package installation"
16+
"## Install Packages"
1717
]
1818
},
1919
{
2020
"cell_type": "code",
21-
"execution_count": null,
21+
"execution_count": 1,
2222
"metadata": {},
2323
"outputs": [],
2424
"source": [
@@ -28,20 +28,18 @@
2828
]
2929
},
3030
{
31-
"cell_type": "code",
32-
"execution_count": null,
31+
"cell_type": "markdown",
3332
"metadata": {},
34-
"outputs": [],
3533
"source": [
36-
"# note that after running the above pip command you should restart the Jupyter kernel\n",
37-
"# to do that click on the restart kernel button next to the \"Code\" button in the upper menu"
34+
"> **Note:** After running the pip command you should restart the Jupyter kernel.<br>\n",
35+
"> To restart the kernel, click on the kernel-restart button in the notebook menu toolbar (the refresh icon next to the **Code** button)."
3836
]
3937
},
4038
{
4139
"cell_type": "markdown",
4240
"metadata": {},
4341
"source": [
44-
"# Import Library"
42+
"## Import Library"
4543
]
4644
},
4745
{
@@ -61,15 +59,15 @@
6159
}
6260
],
6361
"source": [
64-
"# This Python 3 environment comes with many helpful analytics libraries installed\n",
65-
"# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
66-
"# For example, here's several helpful packages to load in \n",
62+
"# This Python 3 environment comes with many helpful analytics libraries installed.\n",
63+
"# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python.\n",
64+
"# For example, here are several helpful packages to load:\n",
6765
"\n",
6866
"import numpy as np # linear algebra\n",
6967
"import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
7068
"\n",
7169
"# Input data files are available in the \"../input/\" directory.\n",
72-
"# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
70+
"# For example, running the following (by selecting 'Run' or pressing Shift+Enter) will list the files in the input directory:\n",
7371
"\n",
7472
"import matplotlib.pyplot as plt\n",
7573
"import random\n",
@@ -106,7 +104,7 @@
106104
"cell_type": "markdown",
107105
"metadata": {},
108106
"source": [
109-
"# Download the data"
107+
"## Download the Data"
110108
]
111109
},
112110
{
@@ -116,7 +114,7 @@
116114
"outputs": [],
117115
"source": [
118116
"!mkdir cats_and_dogs_filtered\n",
119-
"# Download a sample stocks file from Iguazio demo bucket in S3\n",
117+
"# Download a sample stocks file from Iguazio demo bucket in AWS S3\n",
120118
"!curl -L \"iguazio-sample-data.s3.amazonaws.com/catsndogs.zip\" > ./cats_and_dogs_filtered/catsndogs.zip"
121119
]
122120
},
@@ -137,7 +135,7 @@
137135
"_uuid": "7335a579cc0268fba5d34d6f7558f33c187eedb3"
138136
},
139137
"source": [
140-
"# Prepare Traning Data"
138+
"## Prepare the Traning Data"
141139
]
142140
},
143141
{
@@ -168,11 +166,11 @@
168166
},
169167
"outputs": [],
170168
"source": [
171-
"# create filenames list (jpg only)\n",
169+
"# Create a file-names list (JPG image-files only)\n",
172170
"filenames = [file for file in os.listdir(DATA_PATH+\"/cats_n_dogs\") if file.endswith('jpg')]\n",
173171
"categories = []\n",
174172
"\n",
175-
"# categories & prediction classes map\n",
173+
"# Categories and prediction-classes map\n",
176174
"categories_map = {\n",
177175
" 'dog': 1,\n",
178176
" 'cat': 0,\n",
@@ -181,7 +179,7 @@
181179
"with open(MODEL_PATH + 'prediction_classes_map.json', 'w') as f:\n",
182180
" json.dump(prediction_map, f)\n",
183181
"\n",
184-
"# Full samples DF\n",
182+
"# Create a pandas DataFrame for the full sample\n",
185183
"for filename in filenames:\n",
186184
" category = filename.split('.')[0]\n",
187185
" categories.append([categories_map[category]])\n",
@@ -221,7 +219,10 @@
221219
"_uuid": "a999484fc35b73373fafe2253ae9db7ff46fdb90"
222220
},
223221
"source": [
224-
"### See Total In count"
222+
"## Check the Total Image Count\n",
223+
"\n",
224+
"Check the total image count for each category.<br>\n",
225+
"The data set has 12,000 cat images and 12,000 dog images."
225226
]
226227
},
227228
{
@@ -235,22 +236,13 @@
235236
"df['category'].value_counts().plot.bar()"
236237
]
237238
},
238-
{
239-
"cell_type": "markdown",
240-
"metadata": {
241-
"_uuid": "3a08da58107777a1dd05c4a4bf5c484484923cac"
242-
},
243-
"source": [
244-
"From our data we have 12000 cats and 12000 dogs"
245-
]
246-
},
247239
{
248240
"cell_type": "markdown",
249241
"metadata": {
250242
"_uuid": "400a293df3c8499059d9175f3915187074efd971"
251243
},
252244
"source": [
253-
"# See sample image"
245+
"## Display the Sample Image"
254246
]
255247
},
256248
{
@@ -265,13 +257,6 @@
265257
"image = load_img(DATA_PATH+\"/cats_n_dogs/\"+sample)\n",
266258
"plt.imshow(image)"
267259
]
268-
},
269-
{
270-
"cell_type": "code",
271-
"execution_count": null,
272-
"metadata": {},
273-
"outputs": [],
274-
"source": []
275260
}
276261
],
277262
"metadata": {

demos/gpu/horovod-image-classification/02-training-with-horovod-cats-n-dogs.ipynb renamed to demos/gpu/horovod/image-classification/02-train-with-horovod-cats-n-dogs.ipynb

Lines changed: 18 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"cells": [
33
{
44
"cell_type": "code",
5-
"execution_count": null,
5+
"execution_count": 1,
66
"metadata": {},
77
"outputs": [],
88
"source": [
@@ -11,16 +11,16 @@
1111
},
1212
{
1313
"cell_type": "code",
14-
"execution_count": 5,
14+
"execution_count": 2,
1515
"metadata": {},
1616
"outputs": [],
1717
"source": [
18-
"!rm -f /User/horovod/cats_dogs.hd5"
18+
"!rm -f /User/demos/gpu/horovod/image-classification/cats_dogs.hd5"
1919
]
2020
},
2121
{
2222
"cell_type": "code",
23-
"execution_count": 7,
23+
"execution_count": 3,
2424
"metadata": {},
2525
"outputs": [],
2626
"source": [
@@ -29,7 +29,7 @@
2929
},
3030
{
3131
"cell_type": "code",
32-
"execution_count": 8,
32+
"execution_count": 4,
3333
"metadata": {},
3434
"outputs": [
3535
{
@@ -48,9 +48,9 @@
4848
" 'spec': {'replicas': 8,\n",
4949
" 'template': {'spec': {'containers': [{'command': ['mpirun',\n",
5050
" 'python',\n",
51-
" '/User/horovod/hvd_catsndogs.py',\n",
52-
" '/User/demos/image-classification/cats_and_dogs_filtered',\n",
53-
" '/User/horovod'],\n",
51+
" '/User/demos/gpu/horovod/image-classification/horovod_train_cats_n_dogs.py',\n",
52+
" '/User/demos/gpu/horovod/image-classification/cats_and_dogs_filtered',\n",
53+
" '/User/demos/gpu/horovod/image-classification'],\n",
5454
" 'image': 'iguaziodocker/horovod:0.1.1',\n",
5555
" 'name': 'horovod-cats-n-dogs',\n",
5656
" 'resources': {'limits': {'nvidia.com/gpu': 1}},\n",
@@ -68,17 +68,17 @@
6868
"source": [
6969
"from v3io_gputils.mpijob import MpiJob\n",
7070
"\n",
71-
"job = MpiJob(HOROVOD_JOB_NAME, 'iguaziodocker/horovod:0.1.1', ['/User/demos/gpu/image-classification-horovod/hvd_dogncat.py',\n",
72-
" '/User/demos/gpu/image-classification-horvod/cats_and_dogs_filtered',\n",
73-
" '/User/demos/gpu/image-classification-horovod/horovod'])\n",
71+
"job = MpiJob(HOROVOD_JOB_NAME, 'iguaziodocker/horovod:0.1.1', ['/User/demos/gpu/horovod/image-classification/horovod_train_cats_n_dogs.py',\n",
72+
" '/User/demos/gpu/horovod/image-classification/cats_and_dogs_filtered',\n",
73+
" '/User/demos/gpu/horovod/image-classification'])\n",
7474
"\n",
7575
"job.replicas(8).gpus(1)\n",
76-
"job.submit()\n"
76+
"job.submit()"
7777
]
7878
},
7979
{
8080
"cell_type": "code",
81-
"execution_count": 9,
81+
"execution_count": 5,
8282
"metadata": {},
8383
"outputs": [
8484
{
@@ -104,7 +104,7 @@
104104
},
105105
{
106106
"cell_type": "code",
107-
"execution_count": 8,
107+
"execution_count": 6,
108108
"metadata": {},
109109
"outputs": [
110110
{
@@ -132,9 +132,9 @@
132132
" - command:\n",
133133
" - mpirun\n",
134134
" - python\n",
135-
" - /User/horovod/hvd_catsndogs.py\n",
136-
" - /User/demos/image-classification/cats_and_dogs_filtered\n",
137-
" - /User/horovod\n",
135+
" - /User/demos/gpu/horovod/image-classification/horovod_train_cats_n_dogs.py\n",
136+
" - /User/demos/gpu/horovod/image-classification/cats_and_dogs_filtered\n",
137+
" - /User/demos/gpu/horovod/image-classification\n",
138138
" image: iguaziodocker/horovod:0.1.1\n",
139139
" name: horovod-cats-n-dogs\n",
140140
" resources:\n",
@@ -168,7 +168,7 @@
168168
},
169169
{
170170
"cell_type": "code",
171-
"execution_count": 6,
171+
"execution_count": 7,
172172
"metadata": {},
173173
"outputs": [
174174
{
@@ -189,13 +189,6 @@
189189
"source": [
190190
"job.delete()"
191191
]
192-
},
193-
{
194-
"cell_type": "code",
195-
"execution_count": null,
196-
"metadata": {},
197-
"outputs": [],
198-
"source": []
199192
}
200193
],
201194
"metadata": {

0 commit comments

Comments
 (0)