Skip to content

Commit 38e8648

Browse files
Add support for datastore publish (#569)
* Add support for publish to database * Fix testing * Update docs * Make linter happy * Update changelog.md * Add more tests
1 parent 87d0499 commit 38e8648

32 files changed

Lines changed: 701 additions & 37 deletions

README.md

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -119,15 +119,17 @@ Flags:
119119
-c, --config string Configuration file path, defaults: ./wayback.conf, ~/wayback.conf, /etc/wayback.conf
120120
-d, --daemon strings Run as daemon service, supported services are telegram, web, mastodon, twitter, discord, slack, irc, xmpp
121121
--debug Enable debug mode (default mode is false)
122+
--ga Wayback webpages to Ghostarchive (default true)
122123
-h, --help help for wayback
123-
--ia Wayback webpages to Internet Archive
124+
--ia Wayback webpages to Internet Archive (default true)
124125
--info Show application information
125-
--ip Wayback webpages to IPFS
126+
--ip Wayback webpages to IPFS (default true)
126127
--ipfs-host string IPFS daemon host, do not require, unless enable ipfs (default "127.0.0.1")
127128
-m, --ipfs-mode string IPFS mode (default "pinner")
128129
-p, --ipfs-port uint IPFS daemon port (default 5001)
129-
--is Wayback webpages to Archive Today
130-
--ph Wayback webpages to Telegraph
130+
--is Wayback webpages to Archive Today (default true)
131+
--migrate Run SQL migrations
132+
--ph Wayback webpages to Telegraph (default true)
131133
--print Show application configurations
132134
-t, --token string Telegram Bot API Token
133135
--tor Snapshot webpage via Tor anonymity network

cmd/wayback/main.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ var (
4545

4646
configFile string
4747

48+
migrate bool
49+
4850
rootCmd = &cobra.Command{
4951
Use: "wayback",
5052
Short: "A command-line tool and daemon service for archiving webpages.",
@@ -75,7 +77,7 @@ func init() {
7577
rootCmd.Flags().BoolVarP(&ip, "ip", "", true, "Wayback webpages to IPFS")
7678
rootCmd.Flags().BoolVarP(&ph, "ph", "", true, "Wayback webpages to Telegraph")
7779
rootCmd.Flags().BoolVarP(&ga, "ga", "", true, "Wayback webpages to Ghostarchive")
78-
rootCmd.Flags().StringSliceVarP(&daemon, "daemon", "d", []string{}, "Run as daemon service, supported services are telegram, web, mastodon, twitter, discord, slack, irc")
80+
rootCmd.Flags().StringSliceVarP(&daemon, "daemon", "d", []string{}, "Run as daemon service, supported services are telegram, web, mastodon, twitter, discord, slack, irc, xmpp")
7981
rootCmd.Flags().StringVarP(&host, "ipfs-host", "", "127.0.0.1", "IPFS daemon host, do not require, unless enable ipfs")
8082
rootCmd.Flags().UintVarP(&port, "ipfs-port", "p", 5001, "IPFS daemon port")
8183
rootCmd.Flags().StringVarP(&mode, "ipfs-mode", "m", "pinner", "IPFS mode")
@@ -88,6 +90,7 @@ func init() {
8890
rootCmd.Flags().BoolVarP(&debug, "debug", "", false, "Enable debug mode (default mode is false)")
8991
rootCmd.Flags().BoolVarP(&info, "info", "", false, "Show application information")
9092
rootCmd.Flags().BoolVarP(&print, "print", "", false, "Show application configurations")
93+
rootCmd.Flags().BoolVarP(&migrate, "migrate", "", false, "Run SQL migrations")
9194
}
9295

9396
func checkRequiredFlags(cmd *cobra.Command) error {

cmd/wayback/serve.go

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,36 @@ import (
2626
var signalChan chan (os.Signal) = make(chan os.Signal, 1)
2727

2828
func serve(_ *cobra.Command, opts *config.Options, _ []string) {
29-
store, err := storage.Open(opts, "")
29+
db, err := storage.NewConnectionPool(
30+
opts.DatabaseURL(),
31+
opts.DatabaseMinConns(),
32+
opts.DatabaseMaxConns(),
33+
opts.DatabaseConnectionLifetime(),
34+
)
35+
if err != nil {
36+
logger.Fatal("unable to connect to database: %v", err)
37+
}
38+
defer db.Close()
39+
40+
bolt, err := storage.Open(opts, "")
3041
if err != nil {
3142
logger.Fatal("open storage failed: %v", err)
3243
}
44+
store := storage.NewStorage(db, bolt)
3345
defer store.Close()
3446

47+
if !opts.IsDefaultDatabaseURL() {
48+
if err = store.Ping(); err != nil {
49+
logger.Fatal("ping database failed: %v", err)
50+
}
51+
52+
if migrate {
53+
if err = storage.Migrate(db); err != nil {
54+
logger.Fatal("migrate database failed: %v", err)
55+
}
56+
}
57+
}
58+
3559
cfg := []pooling.Option{
3660
pooling.Capacity(opts.PoolingSize()),
3761
pooling.Timeout(opts.WaybackTimeout()),

config/config_test.go

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,190 @@ func TestIPFSMode(t *testing.T) {
243243
}
244244
}
245245

246+
func TestDatabaseURL(t *testing.T) {
247+
var tests = []struct {
248+
url string
249+
expected string
250+
}{
251+
{
252+
url: defDatabaseURL,
253+
expected: defDatabaseURL,
254+
},
255+
{
256+
url: "foo bar",
257+
expected: "foo bar",
258+
},
259+
{
260+
url: "user=username password=passwd host=pg.pooler.com port=6543 dbname=postgres",
261+
expected: "user=username password=passwd host=pg.pooler.com port=6543 dbname=postgres",
262+
},
263+
}
264+
265+
for i, test := range tests {
266+
t.Run(strconv.Itoa(i), func(t *testing.T) {
267+
os.Clearenv()
268+
os.Setenv("WAYBACK_DATABASE_URL", test.url)
269+
270+
parser := NewParser()
271+
opts, err := parser.ParseEnvironmentVariables()
272+
if err != nil {
273+
t.Fatalf(`Parsing environment variables failed: %v`, err)
274+
}
275+
276+
expected := test.expected
277+
got := opts.DatabaseURL()
278+
279+
if got != expected {
280+
t.Errorf(`Unexpected database URL, got %v instead of %s`, got, expected)
281+
}
282+
})
283+
}
284+
}
285+
286+
func TestIsDefaultDatabaseURL(t *testing.T) {
287+
var tests = []struct {
288+
url string
289+
expected bool
290+
}{
291+
{
292+
url: defDatabaseURL,
293+
expected: true,
294+
},
295+
{
296+
url: "foo bar",
297+
expected: false,
298+
},
299+
}
300+
301+
for i, test := range tests {
302+
t.Run(strconv.Itoa(i), func(t *testing.T) {
303+
os.Clearenv()
304+
os.Setenv("WAYBACK_DATABASE_URL", test.url)
305+
306+
parser := NewParser()
307+
opts, err := parser.ParseEnvironmentVariables()
308+
if err != nil {
309+
t.Fatalf(`Parsing environment variables failed: %v`, err)
310+
}
311+
312+
expected := test.expected
313+
got := opts.IsDefaultDatabaseURL()
314+
315+
if got != expected {
316+
t.Errorf(`Unexpected default database URL, got %t instead of %t`, got, expected)
317+
}
318+
})
319+
}
320+
}
321+
322+
func TestDatabaseMaxConns(t *testing.T) {
323+
var tests = []struct {
324+
maxConns int
325+
expected int
326+
}{
327+
{
328+
maxConns: defDatabaseMaxConns,
329+
expected: defDatabaseMaxConns,
330+
},
331+
{
332+
maxConns: 100,
333+
expected: 100,
334+
},
335+
}
336+
337+
for i, test := range tests {
338+
t.Run(strconv.Itoa(i), func(t *testing.T) {
339+
os.Clearenv()
340+
os.Setenv("WAYBACK_DATABASE_MAX_CONNS", strconv.Itoa(test.maxConns))
341+
342+
parser := NewParser()
343+
opts, err := parser.ParseEnvironmentVariables()
344+
if err != nil {
345+
t.Fatalf(`Parsing environment variables failed: %v`, err)
346+
}
347+
348+
expected := test.expected
349+
got := opts.DatabaseMaxConns()
350+
351+
if got != expected {
352+
t.Errorf(`Unexpected maxConns, got %v instead of %d`, got, expected)
353+
}
354+
})
355+
}
356+
}
357+
358+
func TestDatabaseMinConns(t *testing.T) {
359+
var tests = []struct {
360+
minConns int
361+
expected int
362+
}{
363+
{
364+
minConns: defDatabaseMinConns,
365+
expected: defDatabaseMinConns,
366+
},
367+
{
368+
minConns: 100,
369+
expected: 100,
370+
},
371+
}
372+
373+
for i, test := range tests {
374+
t.Run(strconv.Itoa(i), func(t *testing.T) {
375+
os.Clearenv()
376+
os.Setenv("WAYBACK_DATABASE_MIN_CONNS", strconv.Itoa(test.minConns))
377+
378+
parser := NewParser()
379+
opts, err := parser.ParseEnvironmentVariables()
380+
if err != nil {
381+
t.Fatalf(`Parsing environment variables failed: %v`, err)
382+
}
383+
384+
expected := test.expected
385+
got := opts.DatabaseMinConns()
386+
387+
if got != expected {
388+
t.Errorf(`Unexpected minConns, got %v instead of %d`, got, expected)
389+
}
390+
})
391+
}
392+
}
393+
394+
func TestDatabaseConnectionLifetime(t *testing.T) {
395+
var tests = []struct {
396+
connectionLifetime int
397+
expected time.Duration
398+
}{
399+
{
400+
connectionLifetime: defDatabaseConnectionLifetime,
401+
expected: defDatabaseConnectionLifetime * time.Minute,
402+
},
403+
{
404+
connectionLifetime: 100,
405+
expected: 100 * time.Minute,
406+
},
407+
}
408+
409+
for i, test := range tests {
410+
t.Run(strconv.Itoa(i), func(t *testing.T) {
411+
os.Clearenv()
412+
os.Setenv("WAYBACK_DATABASE_CONNECTION_LIFETIME", strconv.Itoa(test.connectionLifetime))
413+
414+
parser := NewParser()
415+
opts, err := parser.ParseEnvironmentVariables()
416+
if err != nil {
417+
t.Fatalf(`Parsing environment variables failed: %v`, err)
418+
}
419+
420+
expected := test.expected
421+
got := opts.DatabaseConnectionLifetime()
422+
423+
if got != expected {
424+
t.Errorf(`Unexpected connection lifetime, got %v instead of %d`, got, expected)
425+
}
426+
})
427+
}
428+
}
429+
246430
func TestIPFSTarget(t *testing.T) {
247431
var tests = []struct {
248432
token string // managed ipfs token

config/options.go

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,8 +104,13 @@ const (
104104
defMeiliApikey = ""
105105

106106
defOmnivoreApikey = ""
107+
defPrivacyURL = ""
107108

108-
defPrivacyURL = ""
109+
defRunMigrations = false
110+
defDatabaseURL = "user=postgres password=postgres dbname=wayback sslmode=disable"
111+
defDatabaseMaxConns = 20
112+
defDatabaseMinConns = 1
113+
defDatabaseConnectionLifetime = 5
109114

110115
maxAttachSizeTelegram = 50000000 // 50MB
111116
maxAttachSizeDiscord = 8000000 // 8MB
@@ -131,6 +136,7 @@ type Options struct {
131136
discord *discord
132137
ipfs *ipfs
133138
slots map[string]bool
139+
database *database
134140
telegram *telegram
135141
mastodon *mastodon
136142
onion *onion
@@ -160,6 +166,13 @@ type Options struct {
160166
waybackFallback bool
161167
}
162168

169+
type database struct {
170+
url string
171+
maxConns int
172+
minConns int
173+
connectionLifetime int
174+
}
175+
163176
type ipfs struct {
164177
host string
165178
mode string
@@ -280,6 +293,12 @@ func NewOptions() *Options {
280293
waybackMaxRetries: defWaybackMaxRetries,
281294
waybackUserAgent: defWaybackUserAgent,
282295
waybackFallback: defWaybackFallback,
296+
database: &database{
297+
url: defDatabaseURL,
298+
maxConns: defDatabaseMaxConns,
299+
minConns: defDatabaseMinConns,
300+
connectionLifetime: defDatabaseConnectionLifetime,
301+
},
283302
ipfs: &ipfs{
284303
host: defIPFSHost,
285304
port: defIPFSPort,
@@ -441,6 +460,31 @@ func (o *Options) EnabledMetrics() bool {
441460
return o.metrics
442461
}
443462

463+
// IsDefaultDatabaseURL returns true if the default database URL is used.
464+
func (o *Options) IsDefaultDatabaseURL() bool {
465+
return o.database.url == defDatabaseURL
466+
}
467+
468+
// DatabaseURL returns the database URL.
469+
func (o *Options) DatabaseURL() string {
470+
return o.database.url
471+
}
472+
473+
// DatabaseMaxConns returns the maximum number of database connections.
474+
func (o *Options) DatabaseMaxConns() int {
475+
return o.database.maxConns
476+
}
477+
478+
// DatabaseMinConns returns the minimum number of database connections.
479+
func (o *Options) DatabaseMinConns() int {
480+
return o.database.minConns
481+
}
482+
483+
// DatabaseConnectionLifetime returns the maximum amount of time a connection may be reused.
484+
func (o *Options) DatabaseConnectionLifetime() time.Duration {
485+
return time.Duration(o.database.connectionLifetime) * time.Minute
486+
}
487+
444488
// IPFSHost returns the host of IPFS daemon service.
445489
func (o *Options) IPFSHost() string {
446490
return o.ipfs.host

config/parser.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,14 @@ func (p *Parser) parseLines(lines []string) (err error) {
9595
p.opts.chromeRemoteAddr = parseString(val, defChromeRemoteAddr)
9696
case "WAYBACK_PROXY":
9797
p.opts.proxy = parseString(val, defProxy)
98+
case "WAYBACK_DATABASE_URL":
99+
p.opts.database.url = parseString(val, defDatabaseURL)
100+
case "WAYBACK_DATABASE_MAX_CONNS":
101+
p.opts.database.maxConns = parseInt(val, defDatabaseMaxConns)
102+
case "WAYBACK_DATABASE_MIN_CONNS":
103+
p.opts.database.minConns = parseInt(val, defDatabaseMinConns)
104+
case "WAYBACK_DATABASE_CONNECTION_LIFETIME":
105+
p.opts.database.connectionLifetime = parseInt(val, defDatabaseConnectionLifetime)
98106
case "WAYBACK_IPFS_HOST":
99107
p.opts.ipfs.host = parseString(val, defIPFSHost)
100108
case "WAYBACK_IPFS_PORT":

docs/changelog.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
99

1010
### Added
1111
- Add support for placing ipfs variables
12+
- Add support for datastore publish ([#569](https://github.com/wabarc/wayback/pull/569))
1213
- Add support publish to Omnivore
1314
- Add privacy notes ([#669](https://github.com/wabarc/wayback/pull/669))
1415

docs/environment.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,10 @@ Use the `-c` / `--config` option to specify the build definition file to use.
4141
| - | `WAYBACK_MEILI_INDEXING` | `capsules` | Meilisearch indexing name |
4242
| - | `WAYBACK_MEILI_APIKEY` | - | Meilisearch admin API key |
4343
| - | `WAYBACK_OMNIVORE_APIKEY` | - | Omnivore API key |
44+
| - | `WAYBACK_DATABASE_URL` | - | The URL of the Postgres database |
45+
| - | `WAYBACK_DATABASE_MAX_CONNS` | `20` | Maximum connections of the Postgres database |
46+
| - | `WAYBACK_DATABASE_MIN_CONNS` | `1` | Minimum connections of the Postgres database |
47+
| - | `WAYBACK_DATABASE_CONNECTION_LIFETIME` | `5` | Connection lifetime of the Postgres database |
4448
| `-d`, `--daemon` | - | - | Run as daemon service, e.g. `telegram`, `web`, `mastodon`, `twitter`, `discord` |
4549
| `--ia` | `WAYBACK_ENABLE_IA` | `true` | Wayback webpages to **Internet Archive** |
4650
| `--is` | `WAYBACK_ENABLE_IS` | `true` | Wayback webpages to **Archive Today** |

0 commit comments

Comments
 (0)