Skip to content

Commit 7c8e016

Browse files
terminaldwellerterminaldweller
committed
added the url to the rss output. fixed a bug with the rss feeds where different rss feeds where not being handled correctly. added new logging functions
1 parent eb841f3 commit 7c8e016

6 files changed

+132
-118
lines changed

‎Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM golang:1.22-alpine3.20 as builder
1+
FROM golang:1.22-alpine3.20 AS builder
22
WORKDIR /milla
33
COPY go.sum go.mod /milla/
44
RUN go mod download

‎Dockerfile_distroless

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM golang:1.22-alpine3.20 as builder
1+
FROM golang:1.22-alpine3.20 AS builder
22
WORKDIR /milla
33
COPY go.sum go.mod /milla/
44
RUN go mod download

‎Dockerfile_distroless_vendored

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM golang:1.22-alpine3.20 as builder
1+
FROM golang:1.22-alpine3.20 AS builder
22
WORKDIR /milla
33
COPY go.sum go.mod /milla/
44
COPY vendor /milla/vendor

‎main.go

+31-35
Original file line numberDiff line numberDiff line change
@@ -1067,49 +1067,45 @@ func ChatGPTHandler(
10671067
}
10681068

10691069
func connectToDB(appConfig *TomlConfig, ctx *context.Context, poolChan chan *pgxpool.Pool) {
1070-
for {
1071-
dbURL := fmt.Sprintf(
1072-
"postgres://%s:%s@%s/%s",
1073-
appConfig.DatabaseUser,
1074-
appConfig.DatabasePassword,
1075-
appConfig.DatabaseAddress,
1076-
appConfig.DatabaseName)
1077-
1078-
log.Println("dbURL:", dbURL)
1070+
dbURL := fmt.Sprintf(
1071+
"postgres://%s:%s@%s/%s",
1072+
appConfig.DatabaseUser,
1073+
appConfig.DatabasePassword,
1074+
appConfig.DatabaseAddress,
1075+
appConfig.DatabaseName)
10791076

1080-
poolConfig, err := pgxpool.ParseConfig(dbURL)
1081-
if err != nil {
1082-
log.Println(err)
1083-
}
1077+
log.Println("dbURL:", dbURL)
10841078

1085-
pool, err := pgxpool.NewWithConfig(*ctx, poolConfig)
1086-
if err != nil {
1087-
log.Println(err)
1088-
time.Sleep(time.Duration(appConfig.MillaReconnectDelay) * time.Second)
1089-
} else {
1090-
log.Printf("%s connected to database", appConfig.IRCDName)
1079+
poolConfig, err := pgxpool.ParseConfig(dbURL)
1080+
if err != nil {
1081+
LogErrorFatal(err)
1082+
}
10911083

1092-
for _, channel := range appConfig.ScrapeChannels {
1093-
tableName := getTableFromChanName(channel, appConfig.IRCDName)
1094-
query := fmt.Sprintf(
1095-
`create table if not exists %s (
1084+
pool, err := pgxpool.NewWithConfig(*ctx, poolConfig)
1085+
if err != nil {
1086+
LogErrorFatal(err)
1087+
} else {
1088+
log.Printf("%s connected to database", appConfig.IRCDName)
1089+
1090+
for _, channel := range appConfig.ScrapeChannels {
1091+
tableName := getTableFromChanName(channel, appConfig.IRCDName)
1092+
query := fmt.Sprintf(
1093+
`create table if not exists %s (
10961094
id serial primary key,
10971095
channel text not null,
10981096
log text not null,
10991097
nick text not null,
11001098
dateadded timestamp default current_timestamp
11011099
)`, tableName)
11021100

1103-
_, err = pool.Exec(*ctx, query)
1104-
if err != nil {
1105-
log.Println(err.Error())
1106-
time.Sleep(time.Duration(appConfig.MillaReconnectDelay) * time.Second)
1107-
}
1101+
_, err = pool.Exec(*ctx, query)
1102+
if err != nil {
1103+
LogErrorFatal(err)
11081104
}
1109-
1110-
appConfig.pool = pool
1111-
poolChan <- pool
11121105
}
1106+
1107+
appConfig.pool = pool
1108+
poolChan <- pool
11131109
}
11141110
}
11151111

@@ -1121,13 +1117,13 @@ func scrapeChannel(irc *girc.Client, poolChan chan *pgxpool.Pool, appConfig Toml
11211117
"insert into %s (channel,log,nick) values ('%s','%s','%s')",
11221118
tableName,
11231119
sanitizeLog(event.Params[0]),
1124-
stripColorCodes(event.Last()),
1120+
sanitizeLog(stripColorCodes(event.Last())),
11251121
event.Source.Name,
11261122
)
11271123

11281124
_, err := pool.Exec(context.Background(), query)
11291125
if err != nil {
1130-
log.Println(err.Error())
1126+
LogError(err)
11311127
}
11321128
})
11331129
}
@@ -1137,7 +1133,7 @@ func populateWatchListWords(appConfig *TomlConfig) {
11371133
for _, filepath := range watchlist.WatchFiles {
11381134
filebytes, err := os.ReadFile(filepath)
11391135
if err != nil {
1140-
log.Println(err.Error())
1136+
LogError(err)
11411137

11421138
continue
11431139
}
@@ -1151,7 +1147,7 @@ func populateWatchListWords(appConfig *TomlConfig) {
11511147
}
11521148
}
11531149

1154-
log.Print(appConfig.WatchLists["security"].Words)
1150+
// log.Print(appConfig.WatchLists["security"].Words)
11551151
}
11561152

11571153
func WatchListHandler(irc *girc.Client, appConfig TomlConfig) {

‎rss.go

+78-80
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ import (
1919
)
2020

2121
func GetFeed(feed FeedConfig,
22-
feedChanel chan<- *gofeed.Feed,
2322
client *girc.Client,
2423
pool *pgxpool.Pool,
2524
channel, groupName string,
@@ -28,104 +27,104 @@ func GetFeed(feed FeedConfig,
2827

2928
parsedFeed, err := feed.FeedParser.ParseURL(feed.URL)
3029
if err != nil {
31-
log.Print(err)
30+
LogError(err)
3231
} else {
33-
query := fmt.Sprintf("select newest_unix_time from rss where name = '%s'", rowName)
34-
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(10)*time.Second)
35-
defer cancel()
32+
if len(parsedFeed.Items) > 0 {
33+
query := fmt.Sprintf("select newest_unix_time from rss where name = '%s'", rowName)
34+
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(10)*time.Second)
35+
defer cancel()
3636

37-
newestFromDB := int64(0)
37+
newestFromDB := int64(0)
3838

39-
err := pool.QueryRow(ctx, query).Scan(&newestFromDB)
40-
if err != nil {
41-
pool.Exec(ctx, fmt.Sprintf("insert into rss (name, newest_unix_time) values ('%s',0)", rowName))
42-
}
39+
err := pool.QueryRow(ctx, query).Scan(&newestFromDB)
40+
if err != nil {
41+
pool.Exec(ctx, fmt.Sprintf("insert into rss (name, newest_unix_time) values ('%s',0)", rowName))
42+
}
4343

44-
log.Print("Newset from DB: ", newestFromDB)
44+
log.Print("Newset from DB: ", newestFromDB)
4545

46-
sortFunc := func(a, b *gofeed.Item) int {
47-
if a.PublishedParsed.Before(*b.PublishedParsed) {
48-
return -1
49-
} else if a.PublishedParsed.After(*b.PublishedParsed) {
50-
return 1
51-
}
46+
sortFunc := func(a, b *gofeed.Item) int {
47+
if a.PublishedParsed.Before(*b.PublishedParsed) {
48+
return -1
49+
} else if a.PublishedParsed.After(*b.PublishedParsed) {
50+
return 1
51+
}
5252

53-
return 0
54-
}
53+
return 0
54+
}
5555

56-
slices.SortFunc(parsedFeed.Items, sortFunc)
56+
slices.SortFunc(parsedFeed.Items, sortFunc)
5757

58-
for _, item := range parsedFeed.Items {
59-
if item.PublishedParsed.Unix() > newestFromDB {
60-
client.Cmd.Message(channel, parsedFeed.Title+": "+item.Title)
58+
for _, item := range parsedFeed.Items {
59+
if item.PublishedParsed.Unix() > newestFromDB {
60+
client.Cmd.Message(channel, parsedFeed.Title+": "+item.Title+">>>"+item.Link)
61+
}
6162
}
62-
}
6363

64-
log.Print(parsedFeed.Items[0].PublishedParsed.Unix())
65-
log.Print(parsedFeed.Items[len(parsedFeed.Items)-1].PublishedParsed.Unix())
64+
query = fmt.Sprintf("update rss set newest_unix_time = %d where name = '%s'", parsedFeed.Items[len(parsedFeed.Items)-1].PublishedParsed.Unix(), rowName)
6665

67-
query = fmt.Sprintf("update rss set newest_unix_time = %d where name = '%s'", parsedFeed.Items[len(parsedFeed.Items)-1].PublishedParsed.Unix(), rowName)
66+
ctx2, cancel := context.WithTimeout(context.Background(), time.Duration(10)*time.Second)
67+
defer cancel()
6868

69-
ctx2, cancel := context.WithTimeout(context.Background(), time.Duration(10)*time.Second)
70-
defer cancel()
69+
_, err = pool.Exec(ctx2, query)
70+
if err != nil {
71+
LogError(err)
72+
}
7173

72-
_, err = pool.Exec(ctx2, query)
73-
if err != nil {
74-
log.Print(err)
7574
}
7675
}
77-
78-
feedChanel <- parsedFeed
7976
}
8077

8178
func feedDispatcher(
8279
config RSSConfig,
8380
client *girc.Client,
8481
pool *pgxpool.Pool,
8582
channel, groupName string,
83+
period int,
8684
) {
87-
feedChanel := make(chan *gofeed.Feed)
85+
for {
86+
for i := range len(config.Feeds) {
87+
config.Feeds[i].FeedParser = gofeed.NewParser()
8888

89-
for i := range len(config.Feeds) {
90-
config.Feeds[i].FeedParser = gofeed.NewParser()
89+
config.Feeds[i].FeedParser.UserAgent = config.Feeds[i].UserAgent
9190

92-
config.Feeds[i].FeedParser.UserAgent = config.Feeds[i].UserAgent
91+
if config.Feeds[i].Proxy != "" {
92+
proxyURL, err := url.Parse(config.Feeds[i].Proxy)
93+
if err != nil {
94+
LogError(err)
9395

94-
if config.Feeds[i].Proxy != "" {
95-
proxyURL, err := url.Parse(config.Feeds[i].Proxy)
96-
if err != nil {
97-
log.Print(err)
98-
continue
99-
}
96+
continue
97+
}
10098

101-
dialer, err := proxy.FromURL(proxyURL, &net.Dialer{Timeout: time.Duration(config.Feeds[i].Timeout) * time.Second})
102-
if err != nil {
103-
log.Print(err)
99+
dialer, err := proxy.FromURL(proxyURL, &net.Dialer{Timeout: time.Duration(config.Feeds[i].Timeout) * time.Second})
100+
if err != nil {
101+
LogError(err)
104102

105-
continue
106-
}
103+
continue
104+
}
107105

108-
httpClient := http.Client{
109-
Transport: &http.Transport{
110-
Dial: dialer.Dial,
111-
},
106+
httpClient := http.Client{
107+
Transport: &http.Transport{
108+
Dial: dialer.Dial,
109+
},
110+
}
111+
112+
config.Feeds[i].FeedParser.Client = &httpClient
112113
}
114+
}
113115

114-
config.Feeds[i].FeedParser.Client = &httpClient
116+
for _, feed := range config.Feeds {
117+
go GetFeed(feed, client, pool, channel, groupName)
115118
}
116-
}
117119

118-
for _, feed := range config.Feeds {
119-
go GetFeed(feed, feedChanel, client, pool, channel, groupName)
120+
time.Sleep(time.Duration(period) * time.Second)
120121
}
121-
122-
// <-feedChanel
123122
}
124123

125124
func ParseRSSConfig(rssConfFilePath string) *RSSConfig {
126125
file, err := os.Open(rssConfFilePath)
127126
if err != nil {
128-
log.Print(err)
127+
LogError(err)
129128

130129
return nil
131130
}
@@ -136,7 +135,7 @@ func ParseRSSConfig(rssConfFilePath string) *RSSConfig {
136135

137136
err = decoder.Decode(&config)
138137
if err != nil {
139-
log.Print(err)
138+
LogError(err)
140139

141140
return nil
142141
}
@@ -145,35 +144,34 @@ func ParseRSSConfig(rssConfFilePath string) *RSSConfig {
145144
}
146145

147146
func runRSS(appConfig *TomlConfig, client *girc.Client) {
148-
for {
149-
query := fmt.Sprintf(
150-
`create table if not exists rss (
147+
query := fmt.Sprintf(
148+
`create table if not exists rss (
151149
id serial primary key,
152150
name text not null unique,
153151
newest_unix_time bigint not null
154152
)`)
155153

156-
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(appConfig.RequestTimeout)*time.Second)
154+
for {
155+
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(10)*time.Second)
157156
defer cancel()
158157

159158
_, err := appConfig.pool.Exec(ctx, query)
160159
if err != nil {
161-
log.Print(err)
162-
time.Sleep(time.Duration(appConfig.MillaReconnectDelay) * time.Second)
160+
LogError(err)
161+
time.Sleep(time.Duration(60) * time.Second)
163162
} else {
164-
for groupName, rss := range appConfig.Rss {
165-
log.Print("RSS: joining ", rss.Channel)
166-
client.Cmd.Join(rss.Channel)
167-
rssConfig := ParseRSSConfig(rss.RssFile)
168-
if rssConfig == nil {
169-
log.Print("Could not parse RSS config file " + rss.RssFile + ". Exiting.")
170-
} else {
171-
for {
172-
feedDispatcher(*rssConfig, client, appConfig.pool, rss.Channel, groupName)
173-
time.Sleep(time.Duration(rssConfig.Period) * time.Second)
174-
}
175-
}
176-
}
163+
break
164+
}
165+
}
166+
167+
for groupName, rss := range appConfig.Rss {
168+
log.Print("RSS: joining ", rss.Channel)
169+
client.Cmd.Join(rss.Channel)
170+
rssConfig := ParseRSSConfig(rss.RssFile)
171+
if rssConfig == nil {
172+
log.Print("Could not parse RSS config file " + rss.RssFile + ". Exiting.")
173+
} else {
174+
go feedDispatcher(*rssConfig, client, appConfig.pool, rss.Channel, groupName, rssConfig.Period)
177175
}
178176
}
179177
}

‎types.go

+20
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package main
22

33
import (
44
"context"
5+
"log"
6+
"runtime"
57
"time"
68

79
"github.com/jackc/pgx/v5/pgxpool"
@@ -191,3 +193,21 @@ type RSSConfig struct {
191193
Feeds []FeedConfig `json:"feeds"`
192194
Period int `json:"period"`
193195
}
196+
197+
func LogError(err error) {
198+
fn, file, line, ok := runtime.Caller(1)
199+
if ok {
200+
log.Printf("%s: %s-%d >>> %v", runtime.FuncForPC(fn).Name(), file, line, err)
201+
} else {
202+
log.Print(err)
203+
}
204+
}
205+
206+
func LogErrorFatal(err error) {
207+
fn, file, line, ok := runtime.Caller(1)
208+
if ok {
209+
log.Fatalf("%s: %s-%d >>> %v", runtime.FuncForPC(fn).Name(), file, line, err)
210+
} else {
211+
log.Fatal(err)
212+
}
213+
}

0 commit comments

Comments
 (0)