Merge pull request #1 from infnetdanpro/develop

release 1.0

Author: infnetdanpro

.gitignore

@@ -4,3 +4,5 @@ venv2
 .json
 .env
 .DS_Store
+*.pyc
+__pycache__

Procfile

@@ -1,2 +1,2 @@
 web: gunicorn app:app
-worker: celery worker --app=lib.celery.app
+worker: celery -A tasks worker --concurrency 1 -E --loglevel=INFO

README.me

@@ -1 +1,22 @@
+# Telegram chat stats bot
+[https://t.me/chat_stats_analytics_bot](https://t.me/chat_stats_analytics_bot)
 
+## How to use?
+0. Start the bot (you can use `/start`).
+1. Export chat history in `json` format
+2. Upload the json-file to chat with bot
+3. Wait the result
+4. Download result (html-format) and enjoy!
+
+## What statistics are provided?
+1. Most common hashtags
+2. List of unique hashtags
+3. List of most replied users
+
+
+## How to add more?
+Use the feedback page: [Google Forms](https://forms.gle/oDhqB5GET79VFiat9)
+
+
+## How to support me?
+Donate links: (donate)[https://www.donationalerts.com/r/infnetdanpro] or BTC: `bc1qdq8nghl0z3tttlwyt3tuer0pe6z382u2kxt47c`

app.py

@@ -1,15 +1,58 @@
-import telebot
+import json
 
-from flask import Flask
+import telebot
+from flask import Flask, request
 
 from config import config
-
+from tasks import run_task
 
 bot = telebot.TeleBot(config["TELEGRAM_TOKEN"])
 app = Flask(__name__)
 
-import bot
+TOKEN = config["TELEGRAM_TOKEN"]
+
+
+@bot.message_handler(commands=["start"])
+def start(message):
+    bot.reply_to(message, "Upload chat history for parsing data")
+
+
+@bot.message_handler(content_types=["document"])
+def handle_docs(message):
+    # file_name = message.document.file_name
+    file_id_info = bot.get_file(message.document.file_id)
+    downloaded_file: bytes = bot.download_file(file_id_info.file_path)
+
+    # in memory
+    file = json.loads(downloaded_file.decode("utf-8"))
+    bot.reply_to(
+        message,
+        "Your data is analyzing. Bot will message you then the result will be ready.",
+    )
+
+    # Put data in Celery App
+    run_task.delay(reply_to=message.from_user.id, data_dict=file)
+
+
+@app.route("/" + TOKEN, methods=["POST"])
+def get_message():
+    json_string = request.get_data().decode("utf-8")
+    update = telebot.types.Update.de_json(json_string)
+    bot.process_new_updates([update])
+    return "!", 200
+
+
+@app.route("/echo", methods=["GET"])
+def echo():
+    return "", 204
 
 
-if __name__ == "__main__":
-    app.run(host="0.0.0.0", port=int(os.environ.get("PORT", 5000)))
+@app.route("/")
+def webhook():
+    bot.remove_webhook()
+    bot.set_webhook(url="https://tg-chat-analytics.herokuapp.com/" + TOKEN)
+    return (
+        '<center><h1><a href="https://t.me/chat_stats_analytics_bot">'
+        "https://t.me/chat_stats_analytics_bot</a></h1></center>",
+        200,
+    )

celeryconfig.py

@@ -1,6 +1,5 @@
 from config import config
 
-CELERY_BROKER_URL = config.get("REDIS_URL", "redis://localhost:6379")
-CELERY_RESULT_BACKEND = config.get("REDIS_URL", "redis://localhost:6379")
-CELERY_TASK_SERIALIZER = "json"
-CELERY_RESULT_SERIALIZER = "json"
+broker_url = config.get("REDIS_URL", "redis://localhost:6379")
+result_backend = config.get("REDIS_URL", "redis://localhost:6379")
+result_serializer = "json"

lib/celery.py

@@ -6,16 +6,23 @@
 import ujson as json
 
 
-class EmptyFilenameException(Exception):
+class JSONLogParserEmptyData(Exception):
     pass
 
 
 class JSONLogParser:
-    def __init__(self, file_name: AnyStr):
-        if not file_name:
-            raise EmptyFilenameException("You should to put correct filename!")
+    def __init__(self, file_name: AnyStr = None, data_dict: dict = None):
+        if not any([file_name, data_dict]):
+            raise JSONLogParserEmptyData(
+                "You should put data into class: file_name or data_dict"
+            )
         self.filename: AnyStr = file_name
-        self.data: Dict = JSONLogParser.parse(file_name)
+        if self.filename:
+            # parse file from disk
+            self.data: Dict = JSONLogParser.parse(file_name)
+        else:
+            # get data from dict
+            self.data = data_dict
         self.name: Union[AnyStr, None] = self.data.get("name")
         self.type: Union[AnyStr, None] = self.data.get("type")
         self.id: Union[int, None] = self.data.get("id")
@@ -144,6 +151,7 @@ def get_user_stats(self):
         self.get_user_messages()
 
         return {
+            "sociable_users": self.sociable_users,
             "users": self.users,
             "messages": self.messages_by_users,
         }
@@ -193,14 +201,20 @@ def clear_text(sentence: str) -> str:
                 messages.extend(text)
         return messages
 
-    def get_words_stats(self, words: List[str]):
+    def get_words_stats(self):
         counter = Counter()
+        words = self.get_all_words()
         morph = pymorphy2.MorphAnalyzer()
         for word in words:
             # TODO: threads here?>
-            counter[morph.parse(word)[0].normal_form] += 1
+            parsed_word = morph.parse(word)[0]
+            if "NOUN" in parsed_word.tag:
+                counter[parsed_word.normal_form] += 1
+            elif "VERB" in parsed_word.tag:
+                counter[parsed_word.normal_form] += 1
 
         self.words_stats = counter
+        return {"word_stats": self.words_stats}
 
     # most replied user
     def get_most_replied_user(self):
@@ -222,15 +236,23 @@ def get_most_replied_user(self):
                     message["id"]
                 ]
 
-        authors_items = []
+        authors_items_users = {}
         for from_id, values in authors_replied_messages.items():
-            authors_items.append(
-                {"from_id": from_id, "count": len(values), "from": users[from_id]}
-            )
-        authors_items = sorted(authors_items, key=lambda x: -x["count"])
-        self.users_replies_stats: List[Dict] = authors_items
+            authors_items_users[users[from_id]] = len(values)
+        self.users_replies_stats: List[Dict] = authors_items_users
         self.users_replies_data: List[Dict] = authors_replied_messages
 
+        return {"most_replies": self.users_replies_stats}
+
+    # main function
+    def generate_stats(self):
+        return {
+            **self.tags_stats(),
+            **self.get_user_stats(),
+            **self.get_words_stats(),
+            **self.get_most_replied_user(),
+        }
+
 
 if __name__ == "__main__":
     filename = "result.json"

lib/stat.py

@@ -0,0 +1,98 @@
+import os
+from io import StringIO
+
+import celery
+import telebot
+from jinja2 import Environment, FileSystemLoader, ChoiceLoader, select_autoescape
+
+from config import config
+from lib.stat import JSONLogParser
+
+app = celery.Celery("celery_app")
+app.config_from_object("celeryconfig")
+
+bot = telebot.TeleBot(config["TELEGRAM_TOKEN"])
+
+path = os.path.join(os.getcwd(), "templates")
+loader = ChoiceLoader([FileSystemLoader(path)])
+env = Environment(
+    loader=loader,
+    autoescape=select_autoescape(),
+)
+
+
+def prepare_template_data(report):
+    hashtags_frequency = {
+        k: v
+        for k, v in sorted(
+            report["hashtags_frequency"].items(), key=lambda item: -item[1]
+        )
+    }
+    hashtags_frequency = {
+        "chart_labels": [f"'{e}'" for e in hashtags_frequency.keys()],
+        "chart_values": [e for e in hashtags_frequency.values()],
+    }
+
+    word_stats = {
+        k: v
+        for k, v in sorted(report["word_stats"].items(), key=lambda item: -item[1])
+        if v > 1
+    }
+    word_stats = {
+        "chart_labels": [f"'{e}'" for e in word_stats.keys()][:250],
+        "chart_values": [e for e in word_stats.values()][:250],
+    }
+
+    most_replies = {
+        k: v
+        for k, v in sorted(report["most_replies"].items(), key=lambda item: -item[1])
+        if v >= 1
+    }
+    most_replies = {
+        "chart_labels": [f"'{e}'" for e in most_replies.keys()][:250],
+        "chart_values": [e for e in most_replies.values()][:250],
+    }
+
+    unique_hashtags = report["unique_hashtags"]
+
+    data = {
+        "hashtags_frequency": {
+            "data": hashtags_frequency,
+            "type": "pie",
+            "title": "Hashtag frequency (all)",
+        },
+        "unique_hashtags": {
+            "data": unique_hashtags,
+            "type": "cloud",
+            "title": "List of unique of hashtags (all)",
+        },
+        "word_stats": {
+            "data": word_stats,
+            "type": "pie",
+            "title": "Word stats (top 250)",
+        },
+        "most_replies": {
+            "data": most_replies,
+            "type": "bar",
+            "title": "Most replied users (top 250)",
+        },
+    }
+
+    return data
+
+
+@app.task
+def run_task(reply_to: int, data_dict: dict):
+    data = JSONLogParser(data_dict=data_dict)
+    report = data.generate_stats()
+    template = env.get_template("report.html")
+
+    # prepare data and render template
+    chart_data = prepare_template_data(report=report)
+    template_data = template.render(chart_data=chart_data, chat_name=data.name)
+
+    obj = StringIO(template_data)
+    obj.name = f"{data.id}.html"
+    bot.send_document(chat_id=reply_to, document=obj, caption="your stats result file")
+
+    # bot.send_message(reply_to, 'Hello!!!')