fcc-ham-or-spam.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/simon-bouchard/freecodecamp-machine-learning-models/blob/main/fcc-ham-or-spam.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "la_Oz6oLlub6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "97cccbba-990e-492a-cda1-e4a947dfc674"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting tf-nightly\n",
            "  Downloading tf_nightly-2.19.0.dev20250128-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.1 kB)\n",
            "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.11/dist-packages (from tf-nightly) (1.4.0)\n",
            "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.11/dist-packages (from tf-nightly) (1.6.3)\n",
            "Requirement already satisfied: flatbuffers>=24.3.25 in /usr/local/lib/python3.11/dist-packages (from tf-nightly) (25.1.21)\n",
            "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /usr/local/lib/python3.11/dist-packages (from tf-nightly) (0.6.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.11/dist-packages (from tf-nightly) (0.2.0)\n",
            "Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.11/dist-packages (from tf-nightly) (18.1.1)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.11/dist-packages (from tf-nightly) (3.4.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from tf-nightly) (24.2)\n",
            "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<6.0.0dev,>=3.20.3 in /usr/local/lib/python3.11/dist-packages (from tf-nightly) (4.25.6)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.11/dist-packages (from tf-nightly) (2.32.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.11/dist-packages (from tf-nightly) (75.1.0)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.11/dist-packages (from tf-nightly) (1.17.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.11/dist-packages (from tf-nightly) (2.5.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.11/dist-packages (from tf-nightly) (4.12.2)\n",
            "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.11/dist-packages (from tf-nightly) (1.17.2)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.11/dist-packages (from tf-nightly) (1.70.0)\n",
            "Collecting tb-nightly~=2.19.0.a (from tf-nightly)\n",
            "  Downloading tb_nightly-2.19.0a20250128-py3-none-any.whl.metadata (1.8 kB)\n",
            "Collecting keras-nightly>=3.6.0.dev (from tf-nightly)\n",
            "  Downloading keras_nightly-3.8.0.dev2025012803-py3-none-any.whl.metadata (6.1 kB)\n",
            "Requirement already satisfied: numpy<2.2.0,>=1.26.0 in /usr/local/lib/python3.11/dist-packages (from tf-nightly) (1.26.4)\n",
            "Requirement already satisfied: h5py>=3.11.0 in /usr/local/lib/python3.11/dist-packages (from tf-nightly) (3.12.1)\n",
            "Requirement already satisfied: ml-dtypes<1.0.0,>=0.4.0 in /usr/local/lib/python3.11/dist-packages (from tf-nightly) (0.4.1)\n",
            "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.11/dist-packages (from tf-nightly) (0.37.1)\n",
            "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.11/dist-packages (from astunparse>=1.6.0->tf-nightly) (0.45.1)\n",
            "Requirement already satisfied: rich in /usr/local/lib/python3.11/dist-packages (from keras-nightly>=3.6.0.dev->tf-nightly) (13.9.4)\n",
            "Requirement already satisfied: namex in /usr/local/lib/python3.11/dist-packages (from keras-nightly>=3.6.0.dev->tf-nightly) (0.0.8)\n",
            "Requirement already satisfied: optree in /usr/local/lib/python3.11/dist-packages (from keras-nightly>=3.6.0.dev->tf-nightly) (0.14.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tf-nightly) (3.4.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tf-nightly) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tf-nightly) (2.3.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tf-nightly) (2024.12.14)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.11/dist-packages (from tb-nightly~=2.19.0.a->tf-nightly) (3.7)\n",
            "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.11/dist-packages (from tb-nightly~=2.19.0.a->tf-nightly) (0.7.2)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from tb-nightly~=2.19.0.a->tf-nightly) (3.1.3)\n",
            "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.11/dist-packages (from werkzeug>=1.0.1->tb-nightly~=2.19.0.a->tf-nightly) (3.0.2)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.11/dist-packages (from rich->keras-nightly>=3.6.0.dev->tf-nightly) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.11/dist-packages (from rich->keras-nightly>=3.6.0.dev->tf-nightly) (2.18.0)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.11/dist-packages (from markdown-it-py>=2.2.0->rich->keras-nightly>=3.6.0.dev->tf-nightly) (0.1.2)\n",
            "Downloading tf_nightly-2.19.0.dev20250128-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (642.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m642.5/642.5 MB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading keras_nightly-3.8.0.dev2025012803-py3-none-any.whl (1.3 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m33.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading tb_nightly-2.19.0a20250128-py3-none-any.whl (5.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.5/5.5 MB\u001b[0m \u001b[31m53.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: tb-nightly, keras-nightly, tf-nightly\n",
            "Successfully installed keras-nightly-3.8.0.dev2025012803 tb-nightly-2.19.0a20250128 tf-nightly-2.19.0.dev20250128\n",
            "Requirement already satisfied: tensorflow-datasets in /usr/local/lib/python3.11/dist-packages (4.9.7)\n",
            "Requirement already satisfied: absl-py in /usr/local/lib/python3.11/dist-packages (from tensorflow-datasets) (1.4.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.11/dist-packages (from tensorflow-datasets) (8.1.8)\n",
            "Requirement already satisfied: dm-tree in /usr/local/lib/python3.11/dist-packages (from tensorflow-datasets) (0.1.8)\n",
            "Requirement already satisfied: immutabledict in /usr/local/lib/python3.11/dist-packages (from tensorflow-datasets) (4.2.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (from tensorflow-datasets) (1.26.4)\n",
            "Requirement already satisfied: promise in /usr/local/lib/python3.11/dist-packages (from tensorflow-datasets) (2.3)\n",
            "Requirement already satisfied: protobuf>=3.20 in /usr/local/lib/python3.11/dist-packages (from tensorflow-datasets) (4.25.6)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.11/dist-packages (from tensorflow-datasets) (5.9.5)\n",
            "Requirement already satisfied: pyarrow in /usr/local/lib/python3.11/dist-packages (from tensorflow-datasets) (17.0.0)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow-datasets) (2.32.3)\n",
            "Requirement already satisfied: simple-parsing in /usr/local/lib/python3.11/dist-packages (from tensorflow-datasets) (0.1.7)\n",
            "Requirement already satisfied: tensorflow-metadata in /usr/local/lib/python3.11/dist-packages (from tensorflow-datasets) (1.16.1)\n",
            "Requirement already satisfied: termcolor in /usr/local/lib/python3.11/dist-packages (from tensorflow-datasets) (2.5.0)\n",
            "Requirement already satisfied: toml in /usr/local/lib/python3.11/dist-packages (from tensorflow-datasets) (0.10.2)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.11/dist-packages (from tensorflow-datasets) (4.67.1)\n",
            "Requirement already satisfied: wrapt in /usr/local/lib/python3.11/dist-packages (from tensorflow-datasets) (1.17.2)\n",
            "Requirement already satisfied: array-record>=0.5.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow-datasets) (0.6.0)\n",
            "Requirement already satisfied: etils>=1.9.1 in /usr/local/lib/python3.11/dist-packages (from etils[edc,enp,epath,epy,etree]>=1.9.1; python_version >= \"3.11\"->tensorflow-datasets) (1.11.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.11/dist-packages (from etils[edc,enp,epath,epy,etree]>=1.9.1; python_version >= \"3.11\"->tensorflow-datasets) (2024.10.0)\n",
            "Requirement already satisfied: importlib_resources in /usr/local/lib/python3.11/dist-packages (from etils[edc,enp,epath,epy,etree]>=1.9.1; python_version >= \"3.11\"->tensorflow-datasets) (6.5.2)\n",
            "Requirement already satisfied: typing_extensions in /usr/local/lib/python3.11/dist-packages (from etils[edc,enp,epath,epy,etree]>=1.9.1; python_version >= \"3.11\"->tensorflow-datasets) (4.12.2)\n",
            "Requirement already satisfied: zipp in /usr/local/lib/python3.11/dist-packages (from etils[edc,enp,epath,epy,etree]>=1.9.1; python_version >= \"3.11\"->tensorflow-datasets) (3.21.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests>=2.19.0->tensorflow-datasets) (3.4.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests>=2.19.0->tensorflow-datasets) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/dist-packages (from requests>=2.19.0->tensorflow-datasets) (2.3.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests>=2.19.0->tensorflow-datasets) (2024.12.14)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.11/dist-packages (from promise->tensorflow-datasets) (1.17.0)\n",
            "Requirement already satisfied: docstring-parser<1.0,>=0.15 in /usr/local/lib/python3.11/dist-packages (from simple-parsing->tensorflow-datasets) (0.16)\n",
            "Requirement already satisfied: googleapis-common-protos<2,>=1.56.4 in /usr/local/lib/python3.11/dist-packages (from tensorflow-metadata->tensorflow-datasets) (1.66.0)\n",
            "2.19.0-dev20250128\n"
          ]
        }
      ],
      "source": [
        "# import libraries\n",
        "try:\n",
        "  # %tensorflow_version only exists in Colab.\n",
        "  !pip install tf-nightly\n",
        "except Exception:\n",
        "  pass\n",
        "import tensorflow as tf\n",
        "import pandas as pd\n",
        "from tensorflow import keras\n",
        "!pip install tensorflow-datasets\n",
        "import tensorflow_datasets as tfds\n",
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from tensorflow.keras import layers, models\n",
        "from tensorflow.keras.models import load_model\n",
        "from tensorflow.keras import Model, Input\n",
        "from tensorflow.keras.layers import Embedding, LSTM, Bidirectional, Dense\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from sklearn.utils.class_weight import compute_class_weight\n",
        "\n",
        "from imblearn.over_sampling import ADASYN\n",
        "\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "\n",
        "print(tf.__version__)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "jaF8r6aOl48C",
        "collapsed": true,
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f606caf8-4be4-437a-b1db-a1c826742a0b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2025-01-28 20:28:36--  https://cdn.freecodecamp.org/project-data/sms/train-data.tsv\n",
            "Resolving cdn.freecodecamp.org (cdn.freecodecamp.org)... 104.26.2.33, 104.26.3.33, 172.67.70.149, ...\n",
            "Connecting to cdn.freecodecamp.org (cdn.freecodecamp.org)|104.26.2.33|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 358233 (350K) [text/tab-separated-values]\n",
            "Saving to: ‘train-data.tsv’\n",
            "\n",
            "\rtrain-data.tsv        0%[                    ]       0  --.-KB/s               \rtrain-data.tsv      100%[===================>] 349.84K  --.-KB/s    in 0.05s   \n",
            "\n",
            "2025-01-28 20:28:37 (7.42 MB/s) - ‘train-data.tsv’ saved [358233/358233]\n",
            "\n",
            "--2025-01-28 20:28:37--  https://cdn.freecodecamp.org/project-data/sms/valid-data.tsv\n",
            "Resolving cdn.freecodecamp.org (cdn.freecodecamp.org)... 104.26.2.33, 104.26.3.33, 172.67.70.149, ...\n",
            "Connecting to cdn.freecodecamp.org (cdn.freecodecamp.org)|104.26.2.33|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 118774 (116K) [text/tab-separated-values]\n",
            "Saving to: ‘valid-data.tsv’\n",
            "\n",
            "valid-data.tsv      100%[===================>] 115.99K  --.-KB/s    in 0.02s   \n",
            "\n",
            "2025-01-28 20:28:37 (5.23 MB/s) - ‘valid-data.tsv’ saved [118774/118774]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# get data files\n",
        "!wget https://cdn.freecodecamp.org/project-data/sms/train-data.tsv\n",
        "!wget https://cdn.freecodecamp.org/project-data/sms/valid-data.tsv\n",
        "\n",
        "train_file_path = \"train-data.tsv\"\n",
        "test_file_path = \"valid-data.tsv\""
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#get data\n",
        "train_data = pd.read_csv('train-data.tsv', sep='\\t', header=None)\n",
        "valid_data = pd.read_csv('valid-data.tsv', sep='\\t', header=None)\n",
        "\n",
        "train_labels = train_data[0]\n",
        "train_texts = train_data[1]\n",
        "\n",
        "valid_labels = valid_data[0]\n",
        "valid_texts = valid_data[1]\n",
        "\n",
        "train_labels.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nVtzV_g4xrV-",
        "outputId": "c7927969-b1eb-4ce2-a341-14870ffa9d00"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(4179,)"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "EOJFeEfumns6"
      },
      "outputs": [],
      "source": [
        "#Preprocessing\n",
        "\n",
        "label_encoder = LabelEncoder()\n",
        "train_labels = label_encoder.fit_transform(train_labels)\n",
        "valid_labels = label_encoder.transform(valid_labels)\n",
        "\n",
        "tokenizer = Tokenizer(oov_token='<oov>')\n",
        "tokenizer.fit_on_texts(train_texts)\n",
        "\n",
        "num_words = len(tokenizer.word_index) + 1\n",
        "max_len = max(max(len(text.split()) for text in train_texts),\n",
        "              max(len(text.split()) for text in valid_texts))\n",
        "\n",
        "train_sequences = tokenizer.texts_to_sequences(train_texts)\n",
        "train_sequences = pad_sequences(train_sequences, padding='post', maxlen=max_len)\n",
        "\n",
        "valid_sequences = tokenizer.texts_to_sequences(valid_texts)\n",
        "valid_sequences = pad_sequences(valid_sequences, padding='post', maxlen=max_len)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "TP0WA8j1mt7Q",
        "collapsed": true,
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 324
        },
        "outputId": "0572fb5d-7ffb-4aaf-b0ef-519dc355445b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.11/dist-packages/keras/src/layers/core/embedding.py:90: UserWarning: Argument `input_length` is deprecated. Just remove it.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1mModel: \"functional_1\"\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"functional_1\"</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
              "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)                        \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape               \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m        Param #\u001b[0m\u001b[1m \u001b[0m┃\n",
              "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
              "│ input_layer_1 (\u001b[38;5;33mInputLayer\u001b[0m)           │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m171\u001b[0m)                 │               \u001b[38;5;34m0\u001b[0m │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ embedding_1 (\u001b[38;5;33mEmbedding\u001b[0m)              │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m171\u001b[0m, \u001b[38;5;34m32\u001b[0m)             │         \u001b[38;5;34m246,208\u001b[0m │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ bidirectional_2 (\u001b[38;5;33mBidirectional\u001b[0m)      │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m171\u001b[0m, \u001b[38;5;34m128\u001b[0m)            │          \u001b[38;5;34m49,664\u001b[0m │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ bidirectional_3 (\u001b[38;5;33mBidirectional\u001b[0m)      │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m64\u001b[0m)                  │          \u001b[38;5;34m41,216\u001b[0m │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ dense_1 (\u001b[38;5;33mDense\u001b[0m)                      │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m)                   │              \u001b[38;5;34m65\u001b[0m │\n",
              "└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
              "┃<span style=\"font-weight: bold\"> Layer (type)                         </span>┃<span style=\"font-weight: bold\"> Output Shape                </span>┃<span style=\"font-weight: bold\">         Param # </span>┃\n",
              "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
              "│ input_layer_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)           │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">171</span>)                 │               <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ embedding_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Embedding</span>)              │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">171</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">32</span>)             │         <span style=\"color: #00af00; text-decoration-color: #00af00\">246,208</span> │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ bidirectional_2 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Bidirectional</span>)      │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">171</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)            │          <span style=\"color: #00af00; text-decoration-color: #00af00\">49,664</span> │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ bidirectional_3 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Bidirectional</span>)      │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)                  │          <span style=\"color: #00af00; text-decoration-color: #00af00\">41,216</span> │\n",
              "├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤\n",
              "│ dense_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)                      │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)                   │              <span style=\"color: #00af00; text-decoration-color: #00af00\">65</span> │\n",
              "└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m337,153\u001b[0m (1.29 MB)\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">337,153</span> (1.29 MB)\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m337,153\u001b[0m (1.29 MB)\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">337,153</span> (1.29 MB)\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "#model\n",
        "inputs = Input(shape=(max_len,))\n",
        "x = Embedding(input_dim=num_words, output_dim=32, input_length=max_len)(inputs)\n",
        "x = Bidirectional(LSTM(64, return_sequences=True))(x)\n",
        "x = Bidirectional(LSTM(32))(x)\n",
        "outputs = Dense(1, activation='sigmoid')(x)\n",
        "\n",
        "model = Model(inputs, outputs)\n",
        "\n",
        "model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
        "\n",
        "model.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "-32RRLY_3voj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8a98bd1e-cfc3-4b5f-c3c9-d4641499b1ae"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/bin/bash: line 1: nvidia-smi: command not found\n",
            "GPU available: []\n"
          ]
        }
      ],
      "source": [
        "!nvidia-smi\n",
        "print(\"GPU available:\", tf.config.list_physical_devices('GPU'))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "pkwq2LFvqabS",
        "collapsed": true,
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b0c00920-c4fe-48e0-9608-5b7b2062731b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Class Weights: {0: 0.5773694390715667, 1: 3.73125}\n",
            "Epoch 1/10\n",
            "\u001b[1m131/131\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m62s\u001b[0m 413ms/step - accuracy: 0.7853 - loss: 0.4372 - val_accuracy: 0.9813 - val_loss: 0.1117\n",
            "Epoch 2/10\n",
            "\u001b[1m131/131\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m79s\u001b[0m 393ms/step - accuracy: 0.7065 - loss: 0.5187 - val_accuracy: 0.9289 - val_loss: 0.1922\n",
            "Epoch 3/10\n",
            "\u001b[1m131/131\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m85s\u001b[0m 417ms/step - accuracy: 0.9482 - loss: 0.1638 - val_accuracy: 0.9145 - val_loss: 0.2287\n",
            "Epoch 4/10\n",
            "\u001b[1m131/131\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m84s\u001b[0m 433ms/step - accuracy: 0.9726 - loss: 0.0985 - val_accuracy: 0.9813 - val_loss: 0.0697\n",
            "Epoch 5/10\n",
            "\u001b[1m131/131\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m79s\u001b[0m 413ms/step - accuracy: 0.9764 - loss: 0.0992 - val_accuracy: 0.9856 - val_loss: 0.0672\n",
            "Epoch 6/10\n",
            "\u001b[1m131/131\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m81s\u001b[0m 407ms/step - accuracy: 0.9898 - loss: 0.0534 - val_accuracy: 0.9842 - val_loss: 0.0726\n",
            "Epoch 7/10\n",
            "\u001b[1m131/131\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m82s\u001b[0m 408ms/step - accuracy: 0.9921 - loss: 0.0600 - val_accuracy: 0.9820 - val_loss: 0.0917\n",
            "Epoch 8/10\n",
            "\u001b[1m131/131\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m80s\u001b[0m 394ms/step - accuracy: 0.9919 - loss: 0.0461 - val_accuracy: 0.9835 - val_loss: 0.0721\n",
            "Epoch 9/10\n",
            "\u001b[1m131/131\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m55s\u001b[0m 417ms/step - accuracy: 0.9949 - loss: 0.0340 - val_accuracy: 0.9842 - val_loss: 0.0835\n",
            "Epoch 10/10\n",
            "\u001b[1m131/131\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m81s\u001b[0m 407ms/step - accuracy: 0.9967 - loss: 0.0203 - val_accuracy: 0.9734 - val_loss: 0.1228\n"
          ]
        }
      ],
      "source": [
        "#Train model\n",
        "#Set class weight (there are more ham than spam)(not necessary)\n",
        "class_weights = compute_class_weight(\n",
        "    class_weight='balanced',\n",
        "    classes=np.unique(train_labels),\n",
        "    y=train_labels\n",
        ")\n",
        "\n",
        "class_weights = dict(enumerate(class_weights))\n",
        "print(f\"Class Weights: {class_weights}\")\n",
        "\n",
        "history = model.fit(\n",
        "    train_sequences,\n",
        "    train_labels,\n",
        "    epochs=10,\n",
        "    batch_size=32,\n",
        "    validation_data=(valid_sequences, valid_labels),\n",
        "    class_weight=class_weights\n",
        "    )\n",
        "\n",
        "model.save('spam_ham_model.keras')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# function to predict messages based on model\n",
        "# (should return list containing prediction and label, ex. [0.008318834938108921, 'ham'])\n",
        "\n",
        "#Uncomment if you don't want to run the previous cells\n",
        "#model = load_model('spam_ham_model.keras')\n",
        "\n",
        "def predict_message(pred_text):\n",
        "\n",
        "  sequence = tokenizer.texts_to_sequences([pred_text])\n",
        "  sequence = pad_sequences(sequence, padding='post', maxlen=max_len)\n",
        "\n",
        "  predictions = model.predict(sequence)\n",
        "\n",
        "  prediction = [predictions[0][0]]\n",
        "\n",
        "  prediction.append('ham' if predictions[0][0] <= 0.5 else 'spam')\n",
        "\n",
        "  return (prediction)\n",
        "\n",
        "pred_text = \"how are you doing today?\"\n",
        "\n",
        "prediction = predict_message(pred_text)\n",
        "print(prediction)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EEVzq7OvAZF1",
        "outputId": "ee074893-f55a-452f-a95e-e2ed6129cd01"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 693ms/step\n",
            "[0.0018118859, 'ham']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "k8aZkwMam4UY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b08c04a1-a4e6-46cd-b239-45a0b8971953"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 79ms/step\n",
            "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 73ms/step\n",
            "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 73ms/step\n",
            "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 73ms/step\n",
            "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 74ms/step\n",
            "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 79ms/step\n",
            "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 73ms/step\n",
            "You passed the challenge. Great job!\n"
          ]
        }
      ],
      "source": [
        "# Run this cell to test your function and model. Do not modify contents.\n",
        "def test_predictions():\n",
        "  test_messages = [\"how are you doing today\",\n",
        "                   \"sale today! to stop texts call 98912460324\",\n",
        "                   \"i dont want to go. can we try it a different day? available sat\",\n",
        "                   \"our new mobile video service is live. just install on your phone to start watching.\",\n",
        "                   \"you have won £1000 cash! call to claim your prize.\",\n",
        "                   \"i'll bring it tomorrow. don't forget the milk.\",\n",
        "                   \"wow, is your arm alright. that happened to me one time too\"\n",
        "                  ]\n",
        "\n",
        "  test_answers = [\"ham\", \"spam\", \"ham\", \"spam\", \"spam\", \"ham\", \"ham\"]\n",
        "  passed = True\n",
        "\n",
        "  for msg, ans in zip(test_messages, test_answers):\n",
        "    prediction = predict_message(msg)\n",
        "    if prediction[1] != ans:\n",
        "      passed = False\n",
        "\n",
        "  if passed:\n",
        "    print(\"You passed the challenge. Great job!\")\n",
        "  else:\n",
        "    print(\"You haven't passed yet. Keep trying.\")\n",
        "\n",
        "test_predictions()\n"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}