Word_Embedding_Example.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Word Embedding Example.ipynb",
      "provenance": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyNekpz2e4Axn6RI4njJ4NnO",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/QasimWani/Market_Analytics/blob/master/Word_Embedding_Example.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6YvKceOpE7v_",
        "colab_type": "code",
        "outputId": "926c9c1d-2fb5-4f3c-e6de-e565300e16f6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "import tensorflow as tf\n",
        "print(tf.__version__)\n",
        "\n",
        "# !pip install -q tensorflow-datasets"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.2.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "skEH0Rn7E9Dd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# This is an example of embedding words to understand the sentiment in a review\n",
        "# this could similarly be used to understand different seenteces and analyze\n",
        "# the user needs accordingly I think.\n",
        "# There this model here can look at a movie review and tell is the movie review is a\n",
        "# good or a bad one. \n",
        "import tensorflow_datasets as tfds\n",
        "imdb, info = tfds.load(\"imdb_reviews\", with_info=True, as_supervised=True)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yEXO52gGF5rc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "train_data, test_data = imdb['train'], imdb['test']\n",
        "\n",
        "training_sentences = []\n",
        "training_labels = []\n",
        "\n",
        "testing_sentences = []\n",
        "testing_labels = []\n",
        "\n",
        "# str(s.tonumpy()) is needed in Python3 instead of just s.numpy()\n",
        "for s,l in train_data:\n",
        "  training_sentences.append(str(s.numpy()))\n",
        "  training_labels.append(l.numpy())\n",
        "  \n",
        "for s,l in test_data:\n",
        "  testing_sentences.append(str(s.numpy()))\n",
        "  testing_labels.append(l.numpy())\n",
        "  \n",
        "training_labels_final = np.array(training_labels)\n",
        "testing_labels_final = np.array(testing_labels)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eQK-fLe4E_VP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "vocab_size = 10000\n",
        "embedding_dim = 16\n",
        "max_length = 120\n",
        "trunc_type='post'\n",
        "oov_tok = \"<OOV>\"\n",
        "\n",
        "\n",
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "tokenizer = Tokenizer(num_words = vocab_size, oov_token=oov_tok)\n",
        "tokenizer.fit_on_texts(training_sentences)\n",
        "word_index = tokenizer.word_index\n",
        "sequences = tokenizer.texts_to_sequences(training_sentences)\n",
        "padded = pad_sequences(sequences,maxlen=max_length, truncating=trunc_type)\n",
        "\n",
        "testing_sequences = tokenizer.texts_to_sequences(testing_sentences)\n",
        "testing_padded = pad_sequences(testing_sequences,maxlen=max_length)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PQHJ91RtF4B9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bTpAri8BFFV9",
        "colab_type": "code",
        "outputId": "0ef0ad09-5759-406a-9c2c-28aee8e1d5dd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        }
      },
      "source": [
        "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n",
        "\n",
        "def decode_review(text):\n",
        "    return ' '.join([reverse_word_index.get(i, '?') for i in text])\n",
        "\n",
        "print(decode_review(padded[3]))\n",
        "print(training_sentences[3])"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? b'this is the kind of film for a snowy sunday afternoon when the rest of the world can go ahead with its own business as you <OOV> into a big arm chair and <OOV> for a couple of hours wonderful performances from cher and nicolas cage as always gently row the plot along there are no <OOV> to cross no dangerous waters just a warm and witty <OOV> through new york life at its best a family film in every sense and one that deserves the praise it received '\n",
            "b'This is the kind of film for a snowy Sunday afternoon when the rest of the world can go ahead with its own business as you descend into a big arm-chair and mellow for a couple of hours. Wonderful performances from Cher and Nicolas Cage (as always) gently row the plot along. There are no rapids to cross, no dangerous waters, just a warm and witty paddle through New York life at its best. A family film in every sense and one that deserves the praise it received.'\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8qk7_36MFVbB",
        "colab_type": "code",
        "outputId": "007f76fa-bee5-44ba-a1f4-542e14e8115b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        }
      },
      "source": [
        "model = tf.keras.Sequential([\n",
        "    tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),\n",
        "    tf.keras.layers.Flatten(),\n",
        "    tf.keras.layers.Dense(6, activation='relu'),\n",
        "    tf.keras.layers.Dense(1, activation='sigmoid')\n",
        "])\n",
        "model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\n",
        "model.summary()"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_1 (Embedding)      (None, 120, 16)           160000    \n",
            "_________________________________________________________________\n",
            "flatten_1 (Flatten)          (None, 1920)              0         \n",
            "_________________________________________________________________\n",
            "dense_2 (Dense)              (None, 6)                 11526     \n",
            "_________________________________________________________________\n",
            "dense_3 (Dense)              (None, 1)                 7         \n",
            "=================================================================\n",
            "Total params: 171,533\n",
            "Trainable params: 171,533\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mw1mnIDiFYw3",
        "colab_type": "code",
        "outputId": "8e54fec0-1558-45c3-e5c4-777954239212",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 748
        }
      },
      "source": [
        "num_epochs = 20\n",
        "model.fit(padded, training_labels_final, epochs=num_epochs, validation_data=(testing_padded, testing_labels_final))"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/20\n",
            "782/782 [==============================] - 4s 5ms/step - loss: 0.4980 - accuracy: 0.7388 - val_loss: 0.3630 - val_accuracy: 0.8392\n",
            "Epoch 2/20\n",
            "782/782 [==============================] - 4s 5ms/step - loss: 0.2452 - accuracy: 0.9038 - val_loss: 0.3662 - val_accuracy: 0.8397\n",
            "Epoch 3/20\n",
            "782/782 [==============================] - 4s 5ms/step - loss: 0.0984 - accuracy: 0.9737 - val_loss: 0.4520 - val_accuracy: 0.8256\n",
            "Epoch 4/20\n",
            "782/782 [==============================] - 4s 5ms/step - loss: 0.0243 - accuracy: 0.9971 - val_loss: 0.5292 - val_accuracy: 0.8272\n",
            "Epoch 5/20\n",
            "782/782 [==============================] - 4s 5ms/step - loss: 0.0072 - accuracy: 0.9993 - val_loss: 0.5942 - val_accuracy: 0.8247\n",
            "Epoch 6/20\n",
            "782/782 [==============================] - 4s 5ms/step - loss: 0.0020 - accuracy: 0.9999 - val_loss: 0.6497 - val_accuracy: 0.8255\n",
            "Epoch 7/20\n",
            "782/782 [==============================] - 4s 5ms/step - loss: 8.6228e-04 - accuracy: 1.0000 - val_loss: 0.6923 - val_accuracy: 0.8254\n",
            "Epoch 8/20\n",
            "782/782 [==============================] - 4s 5ms/step - loss: 4.7418e-04 - accuracy: 1.0000 - val_loss: 0.7339 - val_accuracy: 0.8252\n",
            "Epoch 9/20\n",
            "782/782 [==============================] - 4s 5ms/step - loss: 2.7271e-04 - accuracy: 1.0000 - val_loss: 0.7715 - val_accuracy: 0.8258\n",
            "Epoch 10/20\n",
            "782/782 [==============================] - 4s 5ms/step - loss: 1.6561e-04 - accuracy: 1.0000 - val_loss: 0.8082 - val_accuracy: 0.8258\n",
            "Epoch 11/20\n",
            "782/782 [==============================] - 4s 5ms/step - loss: 1.0084e-04 - accuracy: 1.0000 - val_loss: 0.8452 - val_accuracy: 0.8255\n",
            "Epoch 12/20\n",
            "782/782 [==============================] - 4s 5ms/step - loss: 6.1905e-05 - accuracy: 1.0000 - val_loss: 0.8801 - val_accuracy: 0.8254\n",
            "Epoch 13/20\n",
            "782/782 [==============================] - 4s 5ms/step - loss: 3.8344e-05 - accuracy: 1.0000 - val_loss: 0.9160 - val_accuracy: 0.8258\n",
            "Epoch 14/20\n",
            "782/782 [==============================] - 4s 4ms/step - loss: 2.4137e-05 - accuracy: 1.0000 - val_loss: 0.9507 - val_accuracy: 0.8254\n",
            "Epoch 15/20\n",
            "782/782 [==============================] - 4s 5ms/step - loss: 1.5307e-05 - accuracy: 1.0000 - val_loss: 0.9852 - val_accuracy: 0.8250\n",
            "Epoch 16/20\n",
            "782/782 [==============================] - 4s 5ms/step - loss: 9.7367e-06 - accuracy: 1.0000 - val_loss: 1.0197 - val_accuracy: 0.8253\n",
            "Epoch 17/20\n",
            "782/782 [==============================] - 4s 5ms/step - loss: 6.2974e-06 - accuracy: 1.0000 - val_loss: 1.0552 - val_accuracy: 0.8249\n",
            "Epoch 18/20\n",
            "782/782 [==============================] - 4s 5ms/step - loss: 3.9619e-06 - accuracy: 1.0000 - val_loss: 1.0877 - val_accuracy: 0.8250\n",
            "Epoch 19/20\n",
            "782/782 [==============================] - 4s 5ms/step - loss: 2.5684e-06 - accuracy: 1.0000 - val_loss: 1.1224 - val_accuracy: 0.8247\n",
            "Epoch 20/20\n",
            "782/782 [==============================] - 4s 5ms/step - loss: 1.6443e-06 - accuracy: 1.0000 - val_loss: 1.1558 - val_accuracy: 0.8249\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f807b753c18>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v5NJlpqULNhW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vMF2R9EsGKXl",
        "colab_type": "code",
        "outputId": "b3d08d29-809b-40f0-9a16-e3de8979ec5d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 191
        }
      },
      "source": [
        "e = model.layers[0]\n",
        "weights = e.get_weights()[0]\n",
        "print(weights.shape) # shape: (vocab_size, embedding_dim)\n",
        "Sentence = [\"The movie was a waste of time and money\"]\n",
        "sentence_token = tokenizer.texts_to_sequences(Sentence)\n",
        "print(\"Sentence Token: \" + str(sentence_token))\n",
        "padded_test = pad_sequences(sentence_token,maxlen=max_length, truncating=trunc_type)\n",
        "print(padded_test)\n",
        "model.predict(padded_test)"
      ],
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(10000, 16)\n",
            "Sentence Token: [[2, 18, 14, 4, 446, 5, 60, 3, 277]]\n",
            "[[  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
            "    0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
            "    0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
            "    0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
            "    0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
            "    0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
            "    0   0   0   2  18  14   4 446   5  60   3 277]]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[2.8348773e-06]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3J-iO0g1GN1i",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import io\n",
        "\n",
        "out_v = io.open('vecs.tsv', 'w', encoding='utf-8')\n",
        "out_m = io.open('meta.tsv', 'w', encoding='utf-8')\n",
        "for word_num in range(1, vocab_size):\n",
        "  word = reverse_word_index[word_num]\n",
        "  embeddings = weights[word_num]\n",
        "  out_m.write(word + \"\\n\")\n",
        "  out_v.write('\\t'.join([str(x) for x in embeddings]) + \"\\n\")\n",
        "out_v.close()\n",
        "out_m.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fTL90jVeGSH1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "try:\n",
        "  from google.colab import files\n",
        "except ImportError:\n",
        "  pass\n",
        "else:\n",
        "  files.download('vecs.tsv')\n",
        "  files.download('meta.tsv')"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}