survey_insights_engine.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/rose-deasha/Question/blob/main/survey_insights_engine.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "48a4bfa4-562c-44e2-81eb-e19128462945",
      "metadata": {
        "id": "48a4bfa4-562c-44e2-81eb-e19128462945"
      },
      "outputs": [],
      "source": [
        "import re\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import nltk\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.decomposition import LatentDirichletAllocation, NMF\n",
        "from wordcloud import WordCloud\n",
        "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
        "from gensim.models import LdaModel, CoherenceModel\n",
        "from gensim.corpora import Dictionary\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "from nltk.corpus import stopwords"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "9d391ab6-8897-422e-ba64-02320f1e12e0",
      "metadata": {
        "id": "9d391ab6-8897-422e-ba64-02320f1e12e0"
      },
      "outputs": [],
      "source": [
        "!pip install nltk\n",
        "import nltk\n",
        "nltk.download('vader_lexicon')\n",
        "nltk.download('punkt')\n",
        "nltk.download('wordnet')\n",
        "nltk.download('stopwords')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "dd2b82da-12e0-4de9-ba0e-f09799906a8c",
      "metadata": {
        "id": "dd2b82da-12e0-4de9-ba0e-f09799906a8c"
      },
      "outputs": [],
      "source": [
        "#1 Preprocessing\n",
        "# Initialize lemmatizer and stopwords\n",
        "lemmatizer = WordNetLemmatizer()\n",
        "stop_words = set(stopwords.words('english'))\n",
        "\n",
        "# Add domain-specific stop words\n",
        "domain_stop_words = {'thank', 'please', 'feel', 'best','''I've''','ive','''I'm''','im', 'regards'}\n",
        "stop_words.update(domain_stop_words)\n",
        "\n",
        "def preprocess_text(text):\n",
        "    # Convert to lowercase and remove punctuation\n",
        "    text = re.sub(r'[^\\w\\s]', '', text.lower())\n",
        "    # Tokenize\n",
        "    tokens = word_tokenize(text)\n",
        "    # Remove stopwords and lemmatize\n",
        "    tokens = [lemmatizer.lemmatize(token) for token in tokens if token not in stop_words]\n",
        "    # Remove short words (length < 3)\n",
        "    tokens = [token for token in tokens if len(token) > 2]\n",
        "    return ' '.join(tokens)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "uploaded = files.upload()"
      ],
      "metadata": {
        "id": "aNmw79ghg1Q9"
      },
      "id": "aNmw79ghg1Q9",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "a945b9af-8a94-4f1f-bc7b-94072eeb5311",
      "metadata": {
        "id": "a945b9af-8a94-4f1f-bc7b-94072eeb5311"
      },
      "outputs": [],
      "source": [
        "#grab the data from where it's stored\n",
        "survey_responses = pd.read_csv('Combined_Feedback_Data.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "beed47e6-de67-410a-a6e8-cdf0a6d69039",
      "metadata": {
        "id": "beed47e6-de67-410a-a6e8-cdf0a6d69039"
      },
      "outputs": [],
      "source": [
        "survey_responses.info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8a2ad849-2431-47a5-9d49-4322c13f51c5",
      "metadata": {
        "id": "8a2ad849-2431-47a5-9d49-4322c13f51c5"
      },
      "outputs": [],
      "source": [
        "# Preprocess the survey responses\n",
        "processed_responses = [preprocess_text(response) for response in survey_responses['Feedback']]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "4362c283-ac3b-4572-904b-11b7171f24cb",
      "metadata": {
        "id": "4362c283-ac3b-4572-904b-11b7171f24cb"
      },
      "outputs": [],
      "source": [
        "print(processed_responses)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "4c5d7c8f-c3ca-4110-9ac6-2f446589a94f",
      "metadata": {
        "id": "4c5d7c8f-c3ca-4110-9ac6-2f446589a94f"
      },
      "outputs": [],
      "source": [
        "# Functions\n",
        "# Create a document-term matrix using TF-IDF\n",
        "tfidf = TfidfVectorizer(max_df=0.85, min_df=2, ngram_range=(1, 3))\n",
        "doc_term_matrix = tfidf.fit_transform(processed_responses)\n",
        "\n",
        "feature_names = tfidf.get_feature_names_out()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "35c7e856-8e2b-42dd-b7ce-47d116af010e",
      "metadata": {
        "id": "35c7e856-8e2b-42dd-b7ce-47d116af010e"
      },
      "outputs": [],
      "source": [
        "# Function to print top words for each topic\n",
        "def print_topics(model, feature_names, num_top_words):\n",
        "    for topic_idx, topic in enumerate(model.components_):\n",
        "        top_words = [feature_names[i] for i in topic.argsort()[:-num_top_words - 1:-1]]\n",
        "        print(f\"Topic {topic_idx + 1}: {', '.join(top_words)}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e36a8a73-3c62-4ef3-b395-90d6a2e61e9f",
      "metadata": {
        "id": "e36a8a73-3c62-4ef3-b395-90d6a2e61e9f"
      },
      "outputs": [],
      "source": [
        "# Function to create a single overall word cloud\n",
        "def create_overall_wordcloud(model, feature_names, num_top_words):\n",
        "    # Accumulate top words from all topics\n",
        "    overall_top_words = {}\n",
        "\n",
        "    for topic_idx, topic in enumerate(model.components_):\n",
        "        # Get the top words for the current topic\n",
        "        top_words = {feature_names[i]: topic[i] for i in topic.argsort()[:-num_top_words - 1:-1]}\n",
        "\n",
        "        # Add the word frequencies to the overall dictionary (accumulate across topics)\n",
        "        for word, score in top_words.items():\n",
        "            if word in overall_top_words:\n",
        "                overall_top_words[word] += score  # Sum the importance scores\n",
        "            else:\n",
        "                overall_top_words[word] = score\n",
        "\n",
        "    # Generate the word cloud from the overall accumulated top words\n",
        "    wordcloud = WordCloud(width=1000, height=500, background_color='white').generate_from_frequencies(overall_top_words)\n",
        "\n",
        "    # Display the word cloud\n",
        "    plt.figure(figsize=(12, 8))\n",
        "    plt.imshow(wordcloud, interpolation='bilinear')\n",
        "    plt.axis('off')\n",
        "    plt.savefig('overall_wordcloud.png')\n",
        "    plt.show()\n",
        "    plt.close()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "1e5a9da5-20e9-4a76-94a9-7b09e30f2620",
      "metadata": {
        "id": "1e5a9da5-20e9-4a76-94a9-7b09e30f2620"
      },
      "outputs": [],
      "source": [
        "# Function to create word clouds\n",
        "def create_wordcloud(model, feature_names, num_top_words):\n",
        "    for topic_idx, topic in enumerate(model.components_):\n",
        "        top_words = {feature_names[i]: topic[i] for i in topic.argsort()[:-num_top_words - 1:-1]}\n",
        "        wordcloud = WordCloud(background_color='white').generate_from_frequencies(top_words)\n",
        "\n",
        "        plt.figure(figsize=(8, 6))\n",
        "        plt.imshow(wordcloud, interpolation='bilinear')\n",
        "        plt.axis('off')\n",
        "        plt.title(f'Topic {topic_idx + 1} Word Cloud')\n",
        "        plt.savefig(f'topic_{topic_idx+1}_wordcloud.png')\n",
        "        plt.show()\n",
        "        plt.close()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "06988301-2cd7-4af6-ba2c-b09841eaff88",
      "metadata": {
        "id": "06988301-2cd7-4af6-ba2c-b09841eaff88"
      },
      "outputs": [],
      "source": [
        "# Function to calculate coherence score\n",
        "def calculate_coherence_sklearn(lda_model, texts, feature_names):\n",
        "    # Get topic-word distributions\n",
        "    topic_word_dist = lda_model.components_ / lda_model.components_.sum(axis=1)[:, np.newaxis]\n",
        "\n",
        "    # Convert to gensim format\n",
        "    topics = [[feature_names[i] for i in topic.argsort()[:-11:-1]] for topic in topic_word_dist]\n",
        "\n",
        "    # Create gensim dictionary\n",
        "    dictionary = Dictionary(texts)\n",
        "\n",
        "    # Calculate coherence\n",
        "    cm = CoherenceModel(topics=topics, texts=texts, dictionary=dictionary, coherence='c_v')\n",
        "    return cm.get_coherence()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "9ce7718c-d41f-42dc-901c-a80905763047",
      "metadata": {
        "id": "9ce7718c-d41f-42dc-901c-a80905763047"
      },
      "outputs": [],
      "source": [
        "# Topic Modeling\n",
        "# Perform topic modeling using LDA\n",
        "num_topics = 5\n",
        "lda_model = LatentDirichletAllocation(n_components=num_topics, random_state=42, doc_topic_prior=0.1, topic_word_prior=0.1)\n",
        "lda_output = lda_model.fit_transform(doc_term_matrix)\n",
        "\n",
        "# Print LDA topics\n",
        "print(\"LDA Topics:\")\n",
        "print_topics(lda_model, tfidf.get_feature_names_out(), 10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "509ed90c-26db-4fab-ad7b-5fd486a81c83",
      "metadata": {
        "id": "509ed90c-26db-4fab-ad7b-5fd486a81c83"
      },
      "outputs": [],
      "source": [
        "#Create overall word cloud for LDA topics\n",
        "create_overall_wordcloud(lda_model, feature_names, 10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "60422e0a-2d11-4934-8b5e-841e81e2391b",
      "metadata": {
        "collapsed": true,
        "id": "60422e0a-2d11-4934-8b5e-841e81e2391b"
      },
      "outputs": [],
      "source": [
        "# Create word clouds for LDA topics\n",
        "create_wordcloud(lda_model, tfidf.get_feature_names_out(), 10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ee23345b-5604-41d9-b870-c2680ecf3a5a",
      "metadata": {
        "id": "ee23345b-5604-41d9-b870-c2680ecf3a5a"
      },
      "outputs": [],
      "source": [
        "# Calculate and print LDA coherence score\n",
        "# Ensure processed_responses is tokenized\n",
        "processed_responses1 = [response.split() for response in processed_responses]\n",
        "\n",
        "# Create gensim dictionary\n",
        "dictionary = Dictionary(processed_responses1)\n",
        "\n",
        "# Generate topics using the feature names\n",
        "topics = [[feature_names[i] for i in topic.argsort()[:-11:-1]] for topic in lda_model.components_]\n",
        "\n",
        "# Calculate coherence\n",
        "coherence_model = CoherenceModel(topics=topics, texts=processed_responses1, dictionary=dictionary, coherence='c_v')\n",
        "lda_coherence = coherence_model.get_coherence()\n",
        "\n",
        "print(f\"LDA Coherence Score: {lda_coherence:.2f}\")\n",
        "\n",
        "lda_coherence = calculate_coherence_sklearn(lda_model, processed_responses1, tfidf.get_feature_names_out())\n",
        "print(f\"LDA Coherence Score: {lda_coherence:.2f}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "2fc8f190-21b8-4ba9-818f-2a8e0e4a0593",
      "metadata": {
        "id": "2fc8f190-21b8-4ba9-818f-2a8e0e4a0593"
      },
      "outputs": [],
      "source": [
        "# Perform topic modeling using NMF\n",
        "nmf_model = NMF(n_components=num_topics, random_state=42)\n",
        "nmf_output = nmf_model.fit_transform(doc_term_matrix)\n",
        "\n",
        "# Print NMF topics\n",
        "print(\"\\nNMF Topics:\")\n",
        "print_topics(nmf_model, tfidf.get_feature_names_out(), 10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "9ef13937-7671-4d5c-bec8-cf479da31448",
      "metadata": {
        "id": "9ef13937-7671-4d5c-bec8-cf479da31448"
      },
      "outputs": [],
      "source": [
        "# Create Overall word clouds for NMF topics\n",
        "create_overall_wordcloud(lda_model, tfidf.get_feature_names_out(), 10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5d76bf1a-0dd5-4b97-a500-500427ccc5da",
      "metadata": {
        "id": "5d76bf1a-0dd5-4b97-a500-500427ccc5da",
        "collapsed": true
      },
      "outputs": [],
      "source": [
        "# Create word clouds for NMF topics\n",
        "create_wordcloud(nmf_model, tfidf.get_feature_names_out(), 10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "a868461e-4259-4b1e-8455-ce7a451b4b45",
      "metadata": {
        "id": "a868461e-4259-4b1e-8455-ce7a451b4b45"
      },
      "outputs": [],
      "source": [
        "# Calculate and print NMF coherence score\n",
        "nmf_coherence = calculate_coherence_sklearn(nmf_model, processed_responses1, tfidf.get_feature_names_out())\n",
        "print(f\"NMF Coherence Score: {nmf_coherence:.2f}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "a1f906a9-f8f2-47b8-9451-7aa847225b83",
      "metadata": {
        "id": "a1f906a9-f8f2-47b8-9451-7aa847225b83"
      },
      "outputs": [],
      "source": [
        "# Visualize topic distribution for LDA\n",
        "\n",
        "plt.figure(figsize=(12, 6))\n",
        "topic_proportions = lda_output.mean(axis=0)\n",
        "plt.bar(range(num_topics), topic_proportions)\n",
        "plt.title('Average Topic Distribution in Survey Responses (LDA)')\n",
        "plt.xlabel('Topic')\n",
        "plt.ylabel('Proportion')\n",
        "plt.xticks(range(num_topics), [f'Topic {i+1}' for i in range(num_topics)])\n",
        "plt.savefig('lda_topic_distribution.png')\n",
        "plt.show()\n",
        "plt.close()\n",
        "print(\"\\nTopic distribution chart saved as 'lda_topic_distribution.png'\")\n",
        "\n",
        "# Create a heatmap of topic distribution across responses for LDA\n",
        "plt.figure(figsize=(12, 8))\n",
        "sns.heatmap(lda_output, cmap='YlOrRd', cbar_kws={'label': 'Topic Probability'})\n",
        "plt.title('Topic Distribution Across Survey Responses (LDA)')\n",
        "plt.xlabel('Topics')\n",
        "plt.ylabel('Survey Responses')\n",
        "plt.savefig('lda_topic_heatmap.png')\n",
        "plt.show()\n",
        "plt.close()\n",
        "print(\"Topic heatmap saved as 'lda_topic_heatmap.png'\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "a2713ccc-7e2a-4ffc-a5ea-40e3af739dd2",
      "metadata": {
        "id": "a2713ccc-7e2a-4ffc-a5ea-40e3af739dd2"
      },
      "outputs": [],
      "source": [
        "# Visualize topic distribution for NMF\n",
        "plt.figure(figsize=(12, 6))\n",
        "nmf_topic_proportions = nmf_output.mean(axis=0)\n",
        "plt.bar(range(num_topics), nmf_topic_proportions)\n",
        "plt.title('Average Topic Distribution in Survey Responses (NMF)')\n",
        "plt.xlabel('Topic')\n",
        "plt.ylabel('Proportion')\n",
        "plt.xticks(range(num_topics), [f'Topic {i+1}' for i in range(num_topics)])\n",
        "plt.savefig('nmf_topic_distribution.png')\n",
        "plt.show()\n",
        "plt.close()\n",
        "\n",
        "# Create a heatmap of topic distribution across responses for NMF\n",
        "plt.figure(figsize=(12, 8))\n",
        "sns.heatmap(nmf_output, cmap='YlOrRd', cbar_kws={'label': 'Topic Probability'})\n",
        "plt.title('Topic Distribution Across Survey Responses (NMF)')\n",
        "plt.xlabel('Topics')\n",
        "plt.ylabel('Survey Responses')\n",
        "plt.savefig('nmf_topic_heatmap.png')\n",
        "plt.show()\n",
        "plt.close()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5f236cfe-34f5-4b76-b746-0db83f8d0b4e",
      "metadata": {
        "id": "5f236cfe-34f5-4b76-b746-0db83f8d0b4e"
      },
      "outputs": [],
      "source": [
        "# Initialize the VADER sentiment analyzer\n",
        "sia = SentimentIntensityAnalyzer()\n",
        "\n",
        "# Function to analyze sentiment using VADER\n",
        "def vader_sentiment(text):\n",
        "    score = sia.polarity_scores(text)\n",
        "    return score['compound']\n",
        "\n",
        "# Analyze sentiment using VADER\n",
        "vader_sentiments = [vader_sentiment(response) for response in survey_responses['Feedback']]\n",
        "vader_sentiments_clean = np.nan_to_num(vader_sentiments, nan=0.0, posinf=0.0, neginf=0.0)\n",
        "\n",
        "# Define thresholds for positive, neutral, and negative\n",
        "def categorize_sentiment(score):\n",
        "    if score > 0.1:\n",
        "        return 'Positive'\n",
        "    elif score < -0.1:\n",
        "        return 'Negative'\n",
        "    else:\n",
        "        return 'Neutral'\n",
        "\n",
        "# Apply sentiment categories based on the VADER scores\n",
        "sentiment_categories = [categorize_sentiment(score) for score in vader_sentiments_clean]\n",
        "\n",
        "# Add the sentiment category back into the DataFrame (optional)\n",
        "survey_responses['Sentiment Category'] = sentiment_categories\n",
        "\n",
        "# Calculate the percentage breakdown\n",
        "sentiment_breakdown = pd.Series(sentiment_categories).value_counts(normalize=True) * 100\n",
        "\n",
        "# Print the percentage breakdown for each category\n",
        "print(\"Sentiment breakdown (VADER):\")\n",
        "print(sentiment_breakdown)\n",
        "\n",
        "# Plot sentiment distribution using VADER\n",
        "plt.figure(figsize=(12, 6))\n",
        "sns.histplot(vader_sentiments_clean, bins=20, edgecolor='black')\n",
        "plt.title('Sentiment Distribution of Survey Responses (VADER)')\n",
        "plt.xlabel('Sentiment Score')\n",
        "plt.ylabel('Frequency')\n",
        "plt.savefig('vader_sentiment_distribution.png')\n",
        "plt.show()\n",
        "\n",
        "print(\"VADER sentiment distribution chart saved as 'vader_sentiment_distribution.png'\")\n"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.7"
    },
    "colab": {
      "provenance": [],
      "include_colab_link": true
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}