Merge pull request #51 from slashtechno/customizable-max-backups

Customizable max backups, Ntfy support, and concurrency

Author: slashtechno

.vscode/launch.json

@@ -7,7 +7,7 @@
             "request": "launch",
             "mode": "auto",
             "program": "${workspaceFolder}",
-            "args": ["backup"],
+            "args": ["backup", "--run-type", "${input:runType}"],
             "console": "integratedTerminal"
         },
         {
@@ -16,9 +16,30 @@
             "request": "launch",
             "mode": "auto",
             "program": "${workspaceFolder}",
-            // "args": ["backup", "continuous", "--interval", "5s"],
-            "args": ["backup", "continuous"],
+            // "args": ["backup", "continuous", "--run-type", "${input:runType}", "--interval", "5s", "--max-backups", "5"],
+            "args": ["backup", "continuous", "--run-type", "${input:runType}", "--interval", "${input:interval}", "--max-backups", "${input:maxBackups}"],
             "console": "integratedTerminal"
         },
     ],
+        "inputs": [
+        {
+            "id": "runType",
+            "type": "pickString",
+            "description": "Select the run type for continuous backup",
+            "options": ["clone", "fetch", "dry-run"],
+            "default": "full"
+        },
+        {
+            "id": "interval",
+            "type": "promptString",
+            "description": "Enter the interval for continuous backup",
+            "default": "5s"
+        },
+        {
+            "id": "maxBackups",
+            "type": "promptString",
+            "description": "Enter the maximum number of backups to keep",
+            "default": "2"
+        }
+    ]
 }

README.md

@@ -1,13 +1,15 @@
 # gobackup-github  
-[![Build on multiple platforms](https://github.com/slashtechno/gobackup-github/actions/workflows/go-build.yml/badge.svg)](https://github.com/slashtechno/gobackup-github/actions/workflows/go-build.yml)![GitHub go.mod Go version](https://img.shields.io/github/go-mod/go-version/slashtechno/gobackup-github)[![Create and publish a Docker image](https://github.com/slashtechno/gobackup-github/actions/workflows/docker.yml/badge.svg?branch=main)](https://github.com/slashtechno/gobackup-github/actions/workflows/docker.yml)  
-Go program that utilizes the Github API to backup all a user's repositories, including repositories that have been stored. Multiple users can be backed up, including all members of a GitHub organization.
+[![GitHub Actions Build Workflow Status](https://img.shields.io/github/actions/workflow/status/slashtechno/gobackup-github/go-build.yml?style=for-the-badge&label=Build&labelColor=%2344cc11&color=%23555555)](https://github.com/slashtechno/gobackup-github/actions/workflows/go-build.yml) ![Static Badge](https://img.shields.io/badge/open-source-_?style=for-the-badge&labelColor=%23ef4041&color=%23c13a3a) [![GitHub Actions Docker Workflow Status](https://img.shields.io/github/actions/workflow/status/slashtechno/gobackup-github/docker.yml?style=for-the-badge&label=Docker%20Image%20Build)](https://github.com/slashtechno/gobackup-github/actions/workflows/docker.yml)
+
+Go program that utilizes the Github API to backup a user's repositories, including repositories that have been starred. Multiple users can be backed up, including all members of a GitHub organization.
 ![Demo](demo.gif)
 
 ### Setup and Usage
 1. Create a Github personal access token with the following scopes:  `read:user, repo`  
 2. Either download a binary for your system from releases, or build the program and add it to your PATH with `go install`.
 3. Copy `config.example.yaml` to `config.yaml` and fill in the fields.
     - In addition, command line flags can be used to specify configuration options. Use the `help` command or the `--help` flag for more information.
+    - It is recommended to read through the `config.example.yaml` file to understand the configuration options.
 4. Run the program with `gobackup-github backup` 
     - To perform a rolling backup, run `gobackup-github backup continuous`
 
@@ -20,5 +22,5 @@ To create a container that runs on boot and performs a rolling backup every 24 h
 #### Docker Compose  
 You can also just run `docker compose up -d` to start the container with automatic restarts, assuming you have a `docker-compose.yml` file in the same directory as the `config.yaml` file. You can also edit the `docker-compose.yml` file to change configuration and to manage the rolling backup, if needed.
 
-### Why?  
-I wanted a simple way to backup my Github repositories. I also wanted to learn Go and APIs. Thus, I began this project in 2022 as a way to create my first Go project, use the Github API, and make a utility to backup Github repositories.  
+### Contributing  
+Pull Requests are welcome!  

cassette.tape

@@ -1,3 +1,4 @@
+Output demo.gif
 Type "gobackup-github backup"
 Sleep 500ms
 Enter
@@ -14,4 +15,4 @@ Enter
 Sleep 500ms
 Type "ls backup/slashtechno"
 Enter
-Sleep 5.5s
+Sleep 3.2s

cmd/backup.go

@@ -20,15 +20,18 @@ var backupCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		err := backup.StartBackup(
 			backup.BackupConfig{
-				Usernames:   internal.Viper.GetStringSlice("usernames"),
-				InOrg:       internal.Viper.GetStringSlice("in-org"),
-				BackupStars: internal.Viper.GetBool("stars"),
-				Token:       internal.Viper.GetString("token"),
-				Output:      internal.Viper.GetString("output"),
-				RunType:     internal.Viper.GetString("run-type"),
+				Usernames:         internal.Viper.GetStringSlice("usernames"),
+				InOrg:             internal.Viper.GetStringSlice("in-org"),
+				BackupStars:       internal.Viper.GetBool("stars"),
+				Token:             internal.Viper.GetString("token"),
+				Output:            internal.Viper.GetString("output"),
+				RunType:           internal.Viper.GetString("run-type"),
+				NtfyUrl:           internal.Viper.GetString("ntfy-url"),
+				RecurseSubmodules: internal.Viper.GetUint("recurse-submodules"),
 			},
 			// Pass an empty interval as this is a one-time backup
 			"",
+			0,
 		)
 		if err != nil {
 			log.Error("Backup failed", "err", err)
@@ -67,8 +70,16 @@ func init() {
 	internal.Viper.BindPFlag("backup-stars", backupCmd.PersistentFlags().Lookup("stars"))
 	internal.Viper.SetDefault("backup-stars", false)
 
+	backupCmd.PersistentFlags().Uint("recurse-submodules", 0, "Recurse submodules")
+	internal.Viper.BindPFlag("recurse-submodules", backupCmd.PersistentFlags().Lookup("recurse-submodules"))
+	internal.Viper.SetDefault("recurse-submodules", false)
+
 	backupCmd.PersistentFlags().String("run-type", "", "`Type of backup: clone` (clone the repositories), `fetch` (fetch the repositories and write to output if it ends in .json or `repositories.json` in output), `dry-run` (fetch the repositories and print the output). Default is `clone`")
 	internal.Viper.BindPFlag("run-type", backupCmd.PersistentFlags().Lookup("run-type"))
 	internal.Viper.SetDefault("run-type", "clone")
 
+	backupCmd.PersistentFlags().String("ntfy-url", "", "Ntfy URL to send a notification to after backup")
+	internal.Viper.BindPFlag("ntfy-url", backupCmd.PersistentFlags().Lookup("ntfy-url"))
+	internal.Viper.SetDefault("ntfy-url", "")
+
 }

cmd/continuous.go

@@ -14,19 +14,21 @@ import (
 var continuousCmd = &cobra.Command{
 	Use:   "continuous --interval INTERVAL",
 	Short: "Start a rolling backup that backs up repositories at a set interval",
-	Long:  `Start a rolling backup that backs up repositories at a set interval. The output directory will be emptied each time the backup is run.`,
+	Long:  `Start a rolling backup that backs up repositories at a set interval.`,
 	Run: func(cmd *cobra.Command, args []string) {
 		err := backup.StartBackup(
 			backup.BackupConfig{
-				Usernames:   internal.Viper.GetStringSlice("usernames"),
-				InOrg:       internal.Viper.GetStringSlice("in-org"),
-				BackupStars: internal.Viper.GetBool("stars"),
-				Token:       internal.Viper.GetString("token"),
-				Output:      internal.Viper.GetString("output"),
-				RunType:     internal.Viper.GetString("run-type"),
+				Usernames:         internal.Viper.GetStringSlice("usernames"),
+				InOrg:             internal.Viper.GetStringSlice("in-org"),
+				BackupStars:       internal.Viper.GetBool("stars"),
+				Token:             internal.Viper.GetString("token"),
+				Output:            internal.Viper.GetString("output"),
+				RunType:           internal.Viper.GetString("run-type"),
+				NtfyUrl:           internal.Viper.GetString("ntfy-url"),
+				RecurseSubmodules: internal.Viper.GetUint("recurse-submodules"),
 			},
-			// Pass an empty interval as this is a one-time backup
 			internal.Viper.GetString("interval"),
+			internal.Viper.GetInt("max-backups"),
 		)
 		if err != nil {
 			log.Error("Backup failed", "err", err)
@@ -41,4 +43,8 @@ func init() {
 	internal.Viper.BindPFlag("interval", continuousCmd.Flags().Lookup("interval"))
 	internal.Viper.SetDefault("interval", "24h")
 
+	continuousCmd.Flags().IntP("max-backups", "n", 0, "Number of backups to keep")
+	internal.Viper.BindPFlag("max-backups", continuousCmd.Flags().Lookup("max-backups"))
+	internal.Viper.SetDefault("max-backups", 1)
+
 }

config.example.yaml

@@ -12,7 +12,11 @@ log-level: info
 output: backup
 # GitHub token with read access to the repositories and user
 token: ""
-# List of usernames to fetch. If neither usernames or in-org is set, it will fetch the token owner's repositories
+# List of usernames to fetch. If neither usernames or in-org are set (or an empty string is passed), the authenticated user will (also) be fetched. Fetching the authenticated user also fetches repositories shared with the authenticated user.
 usernames: []
 # `clone` (clone the repositories), `fetch` (fetch the repositories and write to output if it ends in .json or `repositories.json` in output), `dry-run` (fetch the repositories and print the output)
-run-type: clone
+run-type: clone
+# Ntfy URL to optionally send a notification to upon completion. If you don't want to use ntfy.sh, you can use a self-hosted instance of ntfy.
+ntfy-url: ""
+# Submodule depth to include. If set to 0 (default), submodules will not be initialized.
+recurse-submodules: 10

demo.gif

@@ -19,8 +19,11 @@ require (
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/muesli/termenv v0.15.2 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/schollz/progressbar/v3 v3.14.6 // indirect
+	golang.org/x/term v0.23.0 // indirect
 )
 
 require (
@@ -34,6 +37,7 @@ require (
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.5.0 // indirect
+	github.com/go-resty/resty/v2 v2.14.0
 	github.com/gofri/go-github-ratelimit v1.1.0
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
@@ -58,7 +62,7 @@ require (
 	golang.org/x/crypto v0.25.0 // indirect
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
 	golang.org/x/net v0.27.0 // indirect
-	golang.org/x/sys v0.22.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect