added watchlists

Author: terminaldweller

README.md

@@ -263,6 +263,17 @@ The `limit` parameter limits the number of SQL queries that are used to generate
 A `limit` value of 0 disables the limit on the amount of rows that are passed to milla.<br/>
 NOTE: since each milla instance can have its own database, all instances might not necessarily have access to all the data milla is gathering. If you use the same database for all the instances, all instances will have access to all the gathered data.<br/>
 
+## Watchlist
+
+Watchlists allow you to specify a list of channels to watch. The watched values are given in a list of files, each line of the file specifying a value to watch for. Finally a value is given for the alertchannel where the bot will mirror the message that triggered a match.<br/>
+
+```toml
+[ircd.devinet_terra.watchlist.security]
+watchList = ["#securityfeeds"]
+watchFiles = ["/watchfiles/voidbox.list"]
+alertChannel = "#milla_alerts"
+```
+
 ### Example Config File
 
 ```toml
@@ -297,6 +308,10 @@ skipTLSVerify = false
 useTLS = true
 plugins = ["/plugins/plugin1.lua", "/plugins/plugin2.lua"]
 adminOnly = false
+[ircd.devinet.watchlist.security]
+watchList = ["#securityfeeds"]
+watchFiles = ["/watchfiles/voidbox.list"]
+alertChannel = "#milla_alerts"
 
 [ircd.liberanet]
 ircServer = "irc.libera.chat"

main.go

@@ -8,6 +8,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"index/suffixarray"
 	"log"
 	"net"
 	"net/http"
@@ -1044,6 +1045,50 @@ func scrapeChannel(irc *girc.Client, poolChan chan *pgxpool.Pool, appConfig Toml
 	})
 }
 
+func populateWatchListWords(appConfig *TomlConfig) {
+	for watchlistName, watchlist := range appConfig.WatchLists {
+		for _, filepath := range watchlist.WatchFiles {
+			filebytes, err := os.ReadFile(filepath)
+			if err != nil {
+				log.Println(err.Error())
+
+				continue
+			}
+
+			filestring := string(filebytes)
+
+			words := strings.Split(filestring, "\n")
+
+			watchlist.Words = append(watchlist.Words, words...)
+			appConfig.WatchLists[watchlistName] = watchlist
+		}
+	}
+
+	log.Print(appConfig.WatchLists["security"].Words)
+}
+
+func WatchListHandler(irc *girc.Client, appConfig TomlConfig) {
+	irc.Handlers.AddBg(girc.PRIVMSG, func(_ *girc.Client, event girc.Event) {
+		sarray := suffixarray.New([]byte(event.Last()))
+
+		for watchname, watchlist := range appConfig.WatchLists {
+			for _, channel := range watchlist.WatchList {
+				if channel == event.Params[0] {
+					for _, word := range watchlist.Words {
+						indexes := sarray.Lookup([]byte(word), -1)
+						if len(indexes) > 0 {
+							irc.Cmd.Message(watchlist.AlertChannel, fmt.Sprintf("%s: %s", watchname, event.Last()))
+							log.Printf("%s: %s", watchname, event.Last())
+
+							break
+						}
+					}
+				}
+			}
+		}
+	})
+}
+
 func runIRC(appConfig TomlConfig) {
 	var OllamaMemory []MemoryElement
 
@@ -1148,6 +1193,23 @@ func runIRC(appConfig TomlConfig) {
 		go scrapeChannel(irc, poolChan, appConfig)
 	}
 
+	if len(appConfig.WatchLists) > 0 {
+		irc.Handlers.AddBg(girc.CONNECTED, func(client *girc.Client, _ girc.Event) {
+			for _, watchlist := range appConfig.WatchLists {
+				log.Print("joining ", watchlist.AlertChannel)
+				client.Cmd.Join(watchlist.AlertChannel)
+
+				for _, channel := range watchlist.WatchList {
+					client.Cmd.Join(channel)
+				}
+			}
+		})
+
+		populateWatchListWords(&appConfig)
+
+		go WatchListHandler(irc, appConfig)
+	}
+
 	for {
 		var dialer proxy.Dialer
 

scripts/entry_limit_trigger.pgsql

@@ -0,0 +1,27 @@
+create function remove_old_entries()
+    returns trigger
+    as $$
+begin
+    if(
+        select
+            COUNT(*)
+        from
+            table_name) > 10000 then
+        delete from table_name
+        where id in(
+                select
+                    id
+                from
+                    table_name
+                order by
+                    id asc
+                limit 1000);
+    end if;
+    return null;
+end;
+$$
+language plpgsql;
+
+create trigger remove_old_entries_trigger
+    after insert on table_name for each row
+    execute procedure remove_old_entries();

scripts/entry_limit_trigger.sql

@@ -28,6 +28,13 @@ type LuaLstates struct {
 	Cancel   context.CancelFunc
 }
 
+type WatchList struct {
+	AlertChannel string   `toml:"alertChannel"`
+	WatchList    []string `toml:"watchList"`
+	WatchFiles   []string `toml:"watchFiles"`
+	Words        []string `toml:"watchWords"`
+}
+
 type TomlConfig struct {
 	IrcServer           string                   `toml:"ircServer"`
 	IrcNick             string                   `toml:"ircNick"`
@@ -57,6 +64,7 @@ type TomlConfig struct {
 	WebIRCAddress       string                   `toml:"webIRCAddress"`
 	Plugins             []string                 `toml:"plugins"`
 	CustomCommands      map[string]CustomCommand `toml:"customCommands"`
+	WatchLists          map[string]WatchList     `toml:"watchList"`
 	LuaStates           map[string]LuaLstates
 	Temp                float64 `toml:"temp"`
 	RequestTimeout      int     `toml:"requestTimeout"`