Add /stats command

Author: ClockVapor

README.md

@@ -40,6 +40,10 @@ an admin, simply send the command with a user mention following it, and confirm
 `/deleteuserdata @some_user`. As with the `/msg` command, just type the `@` character and select a user from the dropdown that
 opens.
 
+### /stats
+The `/stats` command shows a list of each user's top five most distinguishing words - words they use the most, compared
+to everyone else in the group.
+
 ## Running the Bot
 
 Create a Telegram bot via @BotFather. Take down your bot's access token, and set its privacy mode to disabled so it can
@@ -50,6 +54,7 @@ read all messages in its groups. If privacy mode is enabled, the bot won't be ab
     deletemydata - Delete your Markov chain data in this group
     deletemessagedata - Delete a message from your Markov chain data in this group
     deleteuserdata - (Admin only) Delete a user's Markov chain data in this group
+    stats - Display user statistics
 
 Download the precompiled jar file from the latest [release](https://github.com/ClockVapor/markov-telegram-bot/releases)
 or build it yourself with Maven if you like. If you aren't building it yourself, skip the next paragraph.

pom.xml

@@ -7,7 +7,7 @@
 
     <groupId>clockvapor.telegram.markov-telegram-bot</groupId>
     <artifactId>markov-telegram-bot</artifactId>
-    <version>0.1.1</version>
+    <version>0.2.1</version>
 
     <properties>
         <kotlin.version>1.3.21</kotlin.version>

src/main/kotlin/clockvapor/telegram/markov/MarkovTelegramBot.kt

@@ -113,6 +113,9 @@ class MarkovTelegramBot(private val token: String, private val dataPath: String)
                 matchesCommand(e0Text, "msgall") ->
                     doMessageTotalCommand(bot, message, chatId, text, entities)
 
+                matchesCommand(e0Text, "stats") ->
+                    doStatisticsCommand(bot, message, chatId, text)
+
                 matchesCommand(e0Text, "deletemydata") ->
                     doDeleteMyDataCommand(bot, message, chatId, senderId)
 
@@ -136,6 +139,7 @@ class MarkovTelegramBot(private val token: String, private val dataPath: String)
         val deleteMessageData = wantToDeleteMessageData[chatId]
 
         if (deleteOwnData?.contains(senderId) == true) {
+            bot.sendChatAction(chatId.toLong(), ChatAction.TYPING)
             shouldAnalyzeMessage = false
             deleteOwnData -= senderId
             if (deleteOwnData.isEmpty()) {
@@ -152,6 +156,7 @@ class MarkovTelegramBot(private val token: String, private val dataPath: String)
             }
             reply(bot, message, replyText)
         } else if (deleteUserData?.contains(senderId) == true) {
+            bot.sendChatAction(chatId.toLong(), ChatAction.TYPING)
             shouldAnalyzeMessage = false
             val userIdToDelete = deleteUserData[senderId]!!
             deleteUserData -= senderId
@@ -169,6 +174,7 @@ class MarkovTelegramBot(private val token: String, private val dataPath: String)
             }
             reply(bot, message, replyText)
         } else if (deleteMessageData?.contains(senderId) == true) {
+            bot.sendChatAction(chatId.toLong(), ChatAction.TYPING)
             shouldAnalyzeMessage = false
             val messageToDelete = deleteMessageData[senderId]!!
             deleteMessageData -= senderId
@@ -193,6 +199,7 @@ class MarkovTelegramBot(private val token: String, private val dataPath: String)
     private fun doMessageCommand(bot: Bot, message: Message, chatId: String, text: String,
                                  entities: List<MessageEntity>) {
 
+        bot.sendChatAction(chatId.toLong(), ChatAction.TYPING)
         var parseMode: ParseMode? = null
 
         val replyText = if (entities.size < 2) {
@@ -237,6 +244,7 @@ class MarkovTelegramBot(private val token: String, private val dataPath: String)
     private fun doMessageTotalCommand(bot: Bot, message: Message, chatId: String, text: String,
                                       entities: List<MessageEntity>) {
 
+        bot.sendChatAction(chatId.toLong(), ChatAction.TYPING)
         val e0 = entities[0]
         val remainingTexts = text.substring(e0.offset + e0.length).trim().takeIf { it.isNotBlank() }
             ?.split(whitespaceRegex).orEmpty()
@@ -257,7 +265,32 @@ class MarkovTelegramBot(private val token: String, private val dataPath: String)
         reply(bot, message, replyText)
     }
 
+    private fun doStatisticsCommand(bot: Bot, message: Message, chatId: String, text: String) {
+        bot.sendChatAction(chatId.toLong(), ChatAction.TYPING)
+        val markovPaths = getAllPersonalMarkovPaths(chatId)
+        val userIdToWordCountsMap = markovPaths
+            .mapNotNull { path ->
+                tryOrNull { MarkovChain.read(path) }
+                    ?.let { Pair(File(path).nameWithoutExtension, it.wordCounts) }
+            }
+            .toMap()
+        val universe = computeUniverse(userIdToWordCountsMap.values)
+        val listText = userIdToWordCountsMap.mapNotNull { (userId, wordCounts) ->
+            val response = bot.getChatMember(chatId.toLong(), userId.toLong())
+            val chatMember = response.first?.body()?.result
+            if (chatMember != null) {
+                val mostDistinguishingWords = scoreMostDistinguishingWords(wordCounts, universe).keys.take(5)
+                "${chatMember.user.displayName}\n" +
+                    mostDistinguishingWords.mapIndexed { i, word -> "${i + 1}. $word" }.joinToString("\n")
+            } else null
+        }.filter { it.isNotBlank() }.joinToString("\n\n")
+        val replyText = if (listText.isBlank()) "<no data available>"
+        else "Most distinguishing words:\n\n$listText"
+        reply(bot, message, replyText)
+    }
+
     private fun doDeleteMyDataCommand(bot: Bot, message: Message, chatId: String, senderId: String) {
+        bot.sendChatAction(chatId.toLong(), ChatAction.TYPING)
         wantToDeleteOwnData.getOrPut(chatId) { mutableSetOf() } += senderId
         val replyText = "Are you sure you want to delete your Markov chain data in this group? " +
             "Say \"yes\" to confirm, or anything else to cancel."
@@ -267,6 +300,7 @@ class MarkovTelegramBot(private val token: String, private val dataPath: String)
     private fun doDeleteUserDataCommand(bot: Bot, message: Message, chatId: String, from: User, senderId: String,
                                         entities: List<MessageEntity>) {
 
+        bot.sendChatAction(chatId.toLong(), ChatAction.TYPING)
         var parseMode: ParseMode? = null
         val replyText = if (isAdmin(bot, message.chat, from.id)) {
             if (entities.size < 2) {
@@ -299,6 +333,7 @@ class MarkovTelegramBot(private val token: String, private val dataPath: String)
     }
 
     private fun doDeleteMessageDataCommand(bot: Bot, message: Message, chatId: String, senderId: String) {
+        bot.sendChatAction(chatId.toLong(), ChatAction.TYPING)
         val replyText = message.replyToMessage?.let { replyToMessage ->
             replyToMessage.from?.takeIf { it.id.toString() == senderId }?.let { replyToMessageFrom ->
                 wantToDeleteMessageData.getOrPut(chatId) { mutableMapOf() }[senderId] = replyToMessage.text ?: ""
@@ -413,9 +448,13 @@ class MarkovTelegramBot(private val token: String, private val dataPath: String)
     }
 
     private fun readAllPersonalMarkov(chatId: String): List<MarkovChain> =
+        getAllPersonalMarkovPaths(chatId)
+            .map { MarkovChain.read(it) }
+
+    private fun getAllPersonalMarkovPaths(chatId: String): List<String> =
         File(getChatPath(chatId)).listFiles()
             .filter { !it.name.endsWith("total.json") }
-            .map { MarkovChain.read(it.path) }
+            .map { it.path }
 
     private fun getMarkovPath(chatId: String, userId: String): String =
         Paths.get(getChatPath(chatId), "$userId.json").toString()

src/main/kotlin/clockvapor/telegram/markov/Utils.kt

@@ -0,0 +1,44 @@
+package clockvapor.telegram.markov
+
+import clockvapor.markov.MarkovChain
+
+val MarkovChain.wordCounts: Map<String, Int>
+    get() {
+        val map = hashMapOf<String, Int>()
+        for ((_, dataMap) in data) {
+            for ((word, count) in dataMap) {
+                val sanitized = word.sanitize()
+                if (sanitized.isNotBlank()) {
+                    map.compute(sanitized.toLowerCase()) { _, n -> n?.plus(count) ?: count }
+                }
+            }
+        }
+        return map
+    }
+
+fun scoreMostDistinguishingWords(user: Map<String, Int>, universe: Map<String, Int>): Map<String, Double> {
+    val scores = linkedMapOf<String, Double>()
+    val userTotal = user.values.sum()
+    val universeTotal = universe.values.sum()
+    for ((word, count) in user) {
+        scores[word] = Math.pow(count.toDouble(), 1.1) / userTotal * (universeTotal / universe.getValue(word))
+    }
+    return scores.toList().sortedByDescending { it.second }.toMap()
+}
+
+fun computeUniverse(wordCountsCollection: Collection<Map<String, Int>>): Map<String, Int> {
+    val universe = hashMapOf<String, Int>()
+    for (wordCounts in wordCountsCollection) {
+        for ((word, count) in wordCounts) {
+            universe.compute(word) { _, n -> n?.plus(count) ?: count }
+        }
+    }
+    return universe
+}
+
+private const val punctuation = "[`~!@#$%^&*()\\-_=+\\[\\],<.>/?\\\\|;:\"]+"
+private fun String.sanitize(): String =
+    replace("“", "\"").replace("”", "\"").replace("‘", "'").replace("’", "'")
+        .replace(Regex("^$punctuation"), "")
+        .replace(Regex("$punctuation$"), "")
+