diff --git a/7.1.1-introduction-to-funcational-API.ipynb b/7.1.1-introduction-to-funcational-API.ipynb new file mode 100644 index 0000000000..c8b7aefa87 --- /dev/null +++ b/7.1.1-introduction-to-funcational-API.ipynb @@ -0,0 +1,117 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model: \"sequential\"\n", + "_________________________________________________________________\n", + "Layer (type) Output Shape Param # \n", + "=================================================================\n", + "dense (Dense) (None, 32) 2080 \n", + "_________________________________________________________________\n", + "dense_1 (Dense) (None, 32) 1056 \n", + "_________________________________________________________________\n", + "dense_2 (Dense) (None, 10) 330 \n", + "=================================================================\n", + "Total params: 3,466\n", + "Trainable params: 3,466\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n", + "Model: \"model\"\n", + "_________________________________________________________________\n", + "Layer (type) Output Shape Param # \n", + "=================================================================\n", + "input_1 (InputLayer) [(None, 64)] 0 \n", + "_________________________________________________________________\n", + "dense_3 (Dense) (None, 32) 2080 \n", + "_________________________________________________________________\n", + "dense_4 (Dense) (None, 32) 1056 \n", + "_________________________________________________________________\n", + "dense_5 (Dense) (None, 10) 330 \n", + "=================================================================\n", + "Total params: 3,466\n", + "Trainable params: 3,466\n", + "Non-trainable params: 0\n", + "_________________________________________________________________\n" + ] + } + ], + "source": [ + "from keras.models import Sequential, Model \n", + "from keras import layers\n", + "from keras import Input\n", + "seq_model = Sequential()\n", + "seq_model.add(layers.Dense(32, activation='relu', input_shape=(64,))) \n", + "seq_model.add(layers.Dense(32, activation='relu')) \n", + "seq_model.add(layers.Dense(10, activation='softmax'))\n", + "seq_model.summary()\n", + "\n", + "input_tensor = Input(shape=(64,))\n", + "x = layers.Dense(32, activation='relu')(input_tensor)\n", + "x = layers.Dense(32, activation='relu')(x)\n", + "output_tensor = layers.Dense(10, activation='softmax')(x)\n", + "model = Model(input_tensor, output_tensor) \n", + "model.summary() " + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "ename": "SyntaxError", + "evalue": "invalid syntax (, line 6)", + "output_type": "error", + "traceback": [ + "\u001b[0;36m File \u001b[0;32m\"\"\u001b[0;36m, line \u001b[0;32m6\u001b[0m\n\u001b[0;31m Generates dummy Numpy data to train on\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n" + ] + } + ], + "source": [ + "model.compile(optimizer='rmsprop', loss='categorical_crossentropy')\n", + "\n", + "import numpy as np\n", + "x_train = np.random.random((1000, 64)) \n", + "y_train = np.random.random((1000, 10))\n", + "\n", + "model.fit(x_train, y_train, epochs=10, batch_size=128)\n", + "score = model.evaluate(x_train, y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/7.1.2-build-a-concatenate-network.ipynb b/7.1.2-build-a-concatenate-network.ipynb new file mode 100644 index 0000000000..4d860f1232 --- /dev/null +++ b/7.1.2-build-a-concatenate-network.ipynb @@ -0,0 +1,181 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# two inputs, one output model" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model: \"model_9\"\n", + "__________________________________________________________________________________________________\n", + "Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + "text (InputLayer) [(None, None)] 0 \n", + "__________________________________________________________________________________________________\n", + "question (InputLayer) [(None, None)] 0 \n", + "__________________________________________________________________________________________________\n", + "embedding_20 (Embedding) (None, None, 64) 640000 text[0][0] \n", + "__________________________________________________________________________________________________\n", + "embedding_21 (Embedding) (None, None, 32) 320000 question[0][0] \n", + "__________________________________________________________________________________________________\n", + "lstm_20 (LSTM) (None, 32) 12416 embedding_20[0][0] \n", + "__________________________________________________________________________________________________\n", + "lstm_21 (LSTM) (None, 16) 3136 embedding_21[0][0] \n", + "__________________________________________________________________________________________________\n", + "concatenate_8 (Concatenate) (None, 48) 0 lstm_20[0][0] \n", + " lstm_21[0][0] \n", + "__________________________________________________________________________________________________\n", + "dense_8 (Dense) (None, 500) 24500 concatenate_8[0][0] \n", + "==================================================================================================\n", + "Total params: 1,000,052\n", + "Trainable params: 1,000,052\n", + "Non-trainable params: 0\n", + "__________________________________________________________________________________________________\n", + "None\n" + ] + } + ], + "source": [ + "from keras.models import Model\n", + "from keras import layers\n", + "from keras import Input\n", + "\n", + "text_vocabulary_size = 10000\n", + "question_vocabulary_size = 10000\n", + "answer_vocabulary_size = 500\n", + "\n", + "text_input = Input(shape=(None,), dtype = 'int32', name ='text')\n", + "embedded_text = layers.Embedding(text_vocabulary_size,64)(text_input)\n", + "encoded_text = layers.LSTM(32)(embedded_text)\n", + "\n", + "question_input = Input(shape=(None,), dtype = 'int32', name = 'question')\n", + "embedded_question = layers.Embedding(question_vocabulary_size,32)(question_input)\n", + "encoded_question = layers.LSTM(16)(embedded_question)\n", + "\n", + "# concatenate two inputs and give output\n", + "concatenated = layers.concatenate([encoded_text, encoded_question], axis=-1)\n", + "answer = layers.Dense(answer_vocabulary_size, activation='softmax')(concatenated)\n", + "\n", + "model = Model([text_input, question_input], answer)\n", + "\n", + "model.compile(optimizer = 'rmsprop', loss='categorical_crossentropy', metrics=['acc'])\n", + "print(model.summary())" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1/10\n", + "8/8 [==============================] - 2s 37ms/step - loss: 1553.1508 - acc: 0.0012\n", + "Epoch 2/10\n", + "8/8 [==============================] - 0s 39ms/step - loss: 1556.4614 - acc: 0.0000e+00\n", + "Epoch 3/10\n", + "8/8 [==============================] - 0s 38ms/step - loss: 1562.3912 - acc: 0.0000e+00\n", + "Epoch 4/10\n", + "8/8 [==============================] - 0s 36ms/step - loss: 1568.0669 - acc: 0.0000e+00\n", + "Epoch 5/10\n", + "8/8 [==============================] - 0s 37ms/step - loss: 1570.9047 - acc: 0.0000e+00\n", + "Epoch 6/10\n", + "8/8 [==============================] - 0s 37ms/step - loss: 1569.9131 - acc: 0.0000e+00\n", + "Epoch 7/10\n", + "8/8 [==============================] - 0s 40ms/step - loss: 1575.4770 - acc: 0.0000e+00\n", + "Epoch 8/10\n", + "8/8 [==============================] - 0s 43ms/step - loss: 1572.6059 - acc: 0.0000e+00\n", + "Epoch 9/10\n", + "8/8 [==============================] - 0s 35ms/step - loss: 1572.1957 - acc: 0.0000e+00\n", + "Epoch 10/10\n", + "8/8 [==============================] - 0s 36ms/step - loss: 1572.1025 - acc: 0.0000e+00\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import numpy as np\n", + "\n", + "num_samples = 1000\n", + "max_length = 100\n", + "\n", + "text = np.random.randint(1, text_vocabulary_size, size = (num_samples, max_length))\n", + "\n", + "question = np.random.randint(1, question_vocabulary_size, size = (num_samples, max_length))\n", + "\n", + "answers = np.random.randint(0,2, size = (num_samples, answer_vocabulary_size))\n", + "\n", + "\n", + "model.fit([text, question], answers, epochs = 10, batch_size = 128)\n", + "#model.fit({\"text\":text, \"question\":question}, answers, epochs=10, batch_size=128)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/7.1.3-multi-output-model.ipynb b/7.1.3-multi-output-model.ipynb new file mode 100644 index 0000000000..28cf0c3d43 --- /dev/null +++ b/7.1.3-multi-output-model.ipynb @@ -0,0 +1,177 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# multiple-output models\n", + "## such as a network that takes as input a series of social media posts from a single anonymous person and tries to predict attributes of that person, such as age, gender, and income level " + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "from keras import layers\n", + "from keras import Input\n", + "from keras.models import Model" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "vocabulary_size=50000\n", + "num_income_groups=10\n", + "\n", + "post_input = Input(shape = (None,), dtype=\"int32\", name='posts')\n", + "embedded_posts = layers.Embedding(256, vocabulary_size)(post_input)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "x = layers.Conv1D(128, 5, activation='relu')(embedded_posts)\n", + "x = layers.MaxPooling1D(5)(x)\n", + "x = layers.Conv1D(256, 5, activation='relu')(x)\n", + "x = layers.Conv1D(256, 5, activation='relu')(x)\n", + "x = layers.MaxPooling1D(5)(x)\n", + "x = layers.Conv1D(256, 5, activation='relu')(x)\n", + "x = layers.Conv1D(256, 5, activation='relu')(x)\n", + "x = layers.GlobalMaxPooling1D()(x)\n", + "x = layers.Dense(128, activation='relu')(x)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "age_prediction = layers.Dense(1, name='age')(x)\n", + "income_prediction = layers.Dense(num_income_groups, activation ='softmax', name='income')(x)\n", + "gender_prediction = layers.Dense(1, activation ='sigmoid', name='gender')(x)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model = Model(post_input,[age_prediction, income_prediction, gender_prediction])" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "#To remedy this, you can assign different levels of importance to the loss values in their contribution to the final loss. This is useful in particular if the losses’ values use different scales. For instance, the mean squared error (MSE) loss used for the age-regression task typically takes a value around 3–5, whereas the cross- entropy loss used for the gender-classification task can be as low as 0.1. In such a situa- tion, to balance the contribution of the different losses, you can assign a weight of 10 to the crossentropy loss and a weight of 0.25 to the MSE loss." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "model.compile(optimizer=\"rmsprop\",\n", + " loss = {'age':\"mse\",\n", + " \"income\":\"categorical_crossentropy\",\n", + " \"gender\":\"binary_crossentropy\"},\n", + " loss_weights={\"age\":0.25, \"income\":1, \"gender\":10})" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model: \"model\"\n", + "__________________________________________________________________________________________________\n", + "Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + "posts (InputLayer) [(None, None)] 0 \n", + "__________________________________________________________________________________________________\n", + "embedding (Embedding) (None, None, 50000) 12800000 posts[0][0] \n", + "__________________________________________________________________________________________________\n", + "conv1d (Conv1D) (None, None, 128) 32000128 embedding[0][0] \n", + "__________________________________________________________________________________________________\n", + "max_pooling1d (MaxPooling1D) (None, None, 128) 0 conv1d[0][0] \n", + "__________________________________________________________________________________________________\n", + "conv1d_1 (Conv1D) (None, None, 256) 164096 max_pooling1d[0][0] \n", + "__________________________________________________________________________________________________\n", + "conv1d_2 (Conv1D) (None, None, 256) 327936 conv1d_1[0][0] \n", + "__________________________________________________________________________________________________\n", + "max_pooling1d_1 (MaxPooling1D) (None, None, 256) 0 conv1d_2[0][0] \n", + "__________________________________________________________________________________________________\n", + "conv1d_3 (Conv1D) (None, None, 256) 327936 max_pooling1d_1[0][0] \n", + "__________________________________________________________________________________________________\n", + "conv1d_4 (Conv1D) (None, None, 256) 327936 conv1d_3[0][0] \n", + "__________________________________________________________________________________________________\n", + "global_max_pooling1d (GlobalMax (None, 256) 0 conv1d_4[0][0] \n", + "__________________________________________________________________________________________________\n", + "dense (Dense) (None, 128) 32896 global_max_pooling1d[0][0] \n", + "__________________________________________________________________________________________________\n", + "age (Dense) (None, 1) 129 dense[0][0] \n", + "__________________________________________________________________________________________________\n", + "income (Dense) (None, 10) 1290 dense[0][0] \n", + "__________________________________________________________________________________________________\n", + "gender (Dense) (None, 1) 129 dense[0][0] \n", + "==================================================================================================\n", + "Total params: 45,982,476\n", + "Trainable params: 45,982,476\n", + "Non-trainable params: 0\n", + "__________________________________________________________________________________________________\n" + ] + } + ], + "source": [ + "model.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#model.fit(posts, [age_targets, income_targets, gender_targets], epochs=10, batch_size=64)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/README.md b/README.md index 6039b0d710..0fdae5399b 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,10 @@ These notebooks use Python 3.6 and Keras 2.0.8. They were generated on a p2.xlar * [6.2: Understanding RNNs](http://nbviewer.jupyter.org/github/fchollet/deep-learning-with-python-notebooks/blob/master/6.2-understanding-recurrent-neural-networks.ipynb) * [6.3: Advanced usage of RNNs](http://nbviewer.jupyter.org/github/fchollet/deep-learning-with-python-notebooks/blob/master/6.3-advanced-usage-of-recurrent-neural-networks.ipynb) * [6.4: Sequence processing with convnets](http://nbviewer.jupyter.org/github/fchollet/deep-learning-with-python-notebooks/blob/master/6.4-sequence-processing-with-convnets.ipynb) +* Chapter 7: + * [7.1.1 Introduction to functional API] + * [7.1.2 Build a concatenate network] + * [7.1.3. Multi-output model] * Chapter 8: * [8.1: Text generation with LSTM](http://nbviewer.jupyter.org/github/fchollet/deep-learning-with-python-notebooks/blob/master/8.1-text-generation-with-lstm.ipynb) * [8.2: Deep dream](http://nbviewer.jupyter.org/github/fchollet/deep-learning-with-python-notebooks/blob/master/8.2-deep-dream.ipynb)