Merge pull request #141 from destinygg/mute-link-spam-destiny

Mute users spamming the same link at Destiny.

Author: 11k

lib/configuration/sample.config.json

@@ -22,7 +22,9 @@
     "messagesToKeepPerUser": 10,
     "maxMessagesInList": 5000,
     "timeToLiveSeconds": 600,
-    "tomeStoneIntervalMilliseconds": 60000
+    "tomeStoneIntervalMilliseconds": 60000,
+    "messageUrlSpamSeconds": 600,
+    "messageUrlSpamUser": "Destiny"
   },
   "punishmentCache": {
     "baseMuteSeconds": 60,
@@ -74,7 +76,8 @@
     "messagesToSearch": 75,
     "nukeDepth": 900,
     "minimumStringSearchLength": 100,
-    "uniqueWordsThreshold": 0.45
+    "uniqueWordsThreshold": 0.45,
+    "messageUrlSpamCount": 3
   },
   "roleCache": {
     "timeToLiveSeconds": 21600,

lib/message-routing/message-router.js

@@ -106,6 +106,10 @@ class MessageRouter {
       ),
       this.spamDetection.uniqueWordsCheck(messageContent),
       this.spamDetection.longRepeatedPhrase(messageContent),
+      this.spamDetection.checkListOfRecentUrlsForSpam(
+        messageContent,
+        this.chatCache.getViewerUrlList(user),
+      ),
     ];
 
     if (_.some(spamDetectionList)) {
@@ -117,6 +121,7 @@ class MessageRouter {
         2: 'Too similar to other chatters past text.',
         3: 'Spamming similar phrases too much.',
         4: 'Spamming similar phrases too much.',
+        5: 'Spamming the same link too much.',
       };
 
       this.logger.info({ AUTO_MUTE_REASON: reasons[reasonIndex] }, newMessage);

lib/services/dgg-rolling-chat-cache.js

@@ -5,15 +5,19 @@ const similarity = require('../chat-utils/string-similarity');
 // and then a simple formula to get the change in %
 
 class ChatCache {
-  constructor(config) {
+  constructor(config, messageMatching) {
+    this.messageMatching = messageMatching;
     this.messsagesToKeepPerUser = config.messsagesToKeepPerUser || 2;
     this.maxMessagesInList = config.maxMessagesInList || 2000;
     this.timeToLive = config.timeToLiveSeconds || 1800;
     this.tombStoneInterval = config.tomeStoneIntervalMilliseconds || 120000;
     this.rateLimitMaxMessages = config.rateLimitMaxMessages || 4;
     this.rateLimitSecondsToRefresh = config.rateLimitSecondsToRefresh || 3;
     this.viewerMessageMinimumLength = config.viewerMessageMinimumLength || 20;
+    this.messageUrlSpamSeconds = config.messageUrlSpamSeconds || 600;
+    this.messageUrlSpamUser = config.messageUrlSpamUser || null;
     this.viewerMap = {};
+    this.viewerUrlMap = {};
     this.rateLimitMap = {};
     this.runningMessageList = [];
     this.tombStoneMap = {};
@@ -27,10 +31,18 @@ class ChatCache {
   }
 
   tombStoneCleanup() {
-    if (_.isEmpty(this.tombStoneMap)) {
-      return;
-    }
     const now = moment().unix();
+
+    _.forIn(this.viewerUrlMap, (urls, user) => {
+      const filteredOldUrls = urls.filter((link) => now - link.exp < this.messageUrlSpamSeconds);
+      if (filteredOldUrls.length === 0) {
+        delete this.viewerUrlMap[user];
+      } else {
+        this.viewerUrlMap[user] = filteredOldUrls;
+      }
+    });
+
+    if (_.isEmpty(this.tombStoneMap)) return;
     _.forIn(this.tombStoneMap, (value, key) => {
       if (now - value >= this.timeToLive) {
         delete this.tombStoneMap[key];
@@ -44,6 +56,7 @@ class ChatCache {
     if (message.length >= this.viewerMessageMinimumLength) {
       this.addMessageToViewerMap(user, message);
     }
+    this.addMessageToViewerUrlMap(user, message);
     this.addMessageToRunningList(user, message);
     this.addMessageToRateLimitMap(user);
   }
@@ -62,6 +75,31 @@ class ChatCache {
     this.tombStoneMap[user] = moment().unix();
   }
 
+  addMessageToViewerUrlMap(user, message) {
+    if (
+      this.messageUrlSpamUser &&
+      !this.messageMatching.mentionsUser(message, this.messageUrlSpamUser)
+    ) {
+      return;
+    }
+    this.messageMatching.getLinks(message).forEach((link) => {
+      if (!_.has(this.viewerUrlMap, user)) this.viewerUrlMap[user] = [];
+      this.viewerUrlMap[user].push({
+        url: link.hostname + link.pathname,
+        exp: moment().unix(),
+      });
+    });
+  }
+
+  getViewerUrlList(user) {
+    if (!user || !_.has(this.viewerUrlMap, user)) return [];
+    const now = moment().unix();
+    const filteredOldUrls = this.viewerUrlMap[user]
+      .filter((link) => now - link.exp < this.messageUrlSpamSeconds)
+      .map((link) => link.url);
+    return filteredOldUrls;
+  }
+
   isPastRateLimit(user) {
     const lastMessages = this.rateLimitMap[user];
     if (lastMessages.length === this.rateLimitMaxMessages) {

lib/services/service-index.js

@@ -29,11 +29,15 @@ class Services {
     this.logger = logger(serviceConfigurations.logger);
     this.sql = new Sql(serviceConfigurations.sql);
     this.commandRegistry = new CommandRegistry();
-    this.chatCache = new ChatCache(serviceConfigurations.chatCache);
+    this.messageMatching = messageMatchingService;
+    this.chatCache = new ChatCache(serviceConfigurations.chatCache, this.messageMatching);
     this.punishmentCache = new PunishmentCache(serviceConfigurations.punishmentCache);
     this.roleCache = new RoleCache(serviceConfigurations.roleCache);
     this.punishmentStream = new PunishmentStream(this);
-    this.spamDetection = new SpamDetection(serviceConfigurations.spamDetection);
+    this.spamDetection = new SpamDetection(
+      serviceConfigurations.spamDetection,
+      this.messageMatching,
+    );
     this.scheduledCommands = new ScheduledCommands(serviceConfigurations.schedule);
     this.gulag = gulagService;
     this.lastfm = new LastFm(serviceConfigurations.lastFm);
@@ -43,7 +47,6 @@ class Services {
     this.dggApi = new DggApi(serviceConfigurations.dggApi, this.logger);
     this.twitterApi = new TwitterApi(serviceConfigurations.twitter, this.logger);
     this.messageRelay = new MessageRelay();
-    this.messageMatching = messageMatchingService;
     this.htmlMetadata = new HTMLMetadata();
     // Since reddit relies on managing a single instance of a script, it only runs on the DGG bot.
     if (chatConnectedTo === 'dgg') {

lib/services/spam-detection.js

@@ -14,7 +14,8 @@ const matchStringOrRegex = (message, phrase) => {
   return _.includes(cleanMessage, cleanPhrase.toLowerCase());
 };
 class SpamDetection {
-  constructor(config) {
+  constructor(config, messageMatching) {
+    this.messageMatching = messageMatching;
     this.asciiArtThreshold = config.asciiArtThreshold || 20;
     this.asciiPunctuationCount = config.asciiPunctuationCount || 40;
     this.matchPercentPerUserThreshold = config.matchPercentPerUserThreshold || 0.9;
@@ -26,6 +27,7 @@ class SpamDetection {
     this.uniqueWordsThreshold = config.uniqueWordsThreshold || 0.45;
     this.longWordThreshold = 90;
     this.longWordAllowedSpaces = 4;
+    this.messageUrlSpamCount = config.messageUrlSpamCount || 3;
   }
 
   // Checks whether there's a large number of non ascii characters
@@ -113,6 +115,15 @@ class SpamDetection {
     return this.isSimilarityAboveThreshold(matchPercents);
   }
 
+  checkListOfRecentUrlsForSpam(message, urlList) {
+    return this.messageMatching.getLinks(message).some((link) => {
+      return (
+        urlList.filter((url) => url === link.hostname + link.pathname).length >=
+        this.messageUrlSpamCount - 1
+      );
+    });
+  }
+
   static isMessageNuked(nukedPhrases, messageContent) {
     let nukeDuration = 0;
     let nukePhrase = '';