Properly (re-)initialize config subsystem during the tests

In some unit tests, we need to alter the configuration via env var
(to ensure the environment-based changes are effective); this adds
in re-initialization at the appropriate points.

Additionally, this moves the various timeout constants to the config
subsystem, allowing us to use much shorter timeouts for test runs
(taking about 6 minutes off the time to run `go test`).

Author: bbockelm

config/config.go

@@ -2,7 +2,7 @@
 package config
 
 import (
-	"path"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"os"
@@ -43,7 +43,7 @@ type OSDFConfig struct {
 // need to manually configure the location of the director endpoint.
 //
 func GetPreferredPrefix() string {
-	arg0 := strings.ToUpper(path.Base(os.Args[0]))
+	arg0 := strings.ToUpper(filepath.Base(os.Args[0]))
 	underscore_idx := strings.Index(arg0, "_")
 	if underscore_idx != -1 {
 		return arg0[0:underscore_idx]
@@ -76,8 +76,12 @@ func Init() error {
 	upper_prefix := GetPreferredPrefix()
 	lower_prefix := strings.ToLower(upper_prefix)
 
+	viper.SetDefault("StoppedTransferTimeout", 100)
+	viper.SetDefault("SlowTransferRampupTime", 100)
+	viper.SetDefault("SlowTransferWindow", 30)
+
 	if upper_prefix == "OSDF" || upper_prefix == "STASH" {
-		viper.Set("NamespaceURL", "https://topology.opensciencegrid.org/stashcache/namespaces")
+		viper.SetDefault("NamespaceURL", "https://topology.opensciencegrid.org/stashcache/namespaces")
 	}
 
 	viper.SetEnvPrefix(upper_prefix)

handle_http.go

@@ -488,6 +488,9 @@ func DownloadHTTP(transfer TransferDetails, dest string, token string) (int64, e
 		)
 	}
 
+	stoppedTransferTimeout := viper.GetInt64("StoppedTransferTimeout")
+	slowTransferRampupTime := viper.GetInt64("SlowTransferRampupTime")
+	slowTransferWindow := viper.GetInt64("SlowTransferWindow")
 	var previousCompletedBytes int64 = 0
 	var previousCompletedTime = time.Now()
 	var startBelowLimit int64 = 0
@@ -514,8 +517,8 @@ Loop:
 			if resp.BytesComplete() == lastBytesComplete {
 				if noProgressStartTime.IsZero() {
 					noProgressStartTime = time.Now()
-				} else if time.Since(noProgressStartTime) > 100 * time.Second {
-					errMsg := "No progress for more than " + (time.Since(noProgressStartTime)/time.Second).String() + " seconds."
+				} else if time.Since(noProgressStartTime) > time.Duration(stoppedTransferTimeout) * time.Second {
+					errMsg := "No progress for more than " + time.Since(noProgressStartTime).Truncate(time.Millisecond).String()
 					log.Errorln(errMsg)
 					return 5, &StoppedTransferError{
 						Err: errMsg,
@@ -529,18 +532,18 @@ Loop:
 
 			// Check if we are downloading fast enough
 			if resp.BytesPerSecond() < float64(downloadLimit) {
-				// Give the download 120 seconds to start
-				if resp.Duration() < time.Second*120 {
+				// Give the download `slowTransferRampupTime` (default 120) seconds to start
+				if resp.Duration() < time.Second* time.Duration(slowTransferRampupTime) {
 					continue
 				} else if startBelowLimit == 0 {
 					log.Warnln("Download speed of ", resp.BytesPerSecond(), "bytes/s", " is below the limit of", downloadLimit, "bytes/s")
 					startBelowLimit = time.Now().Unix()
 					continue
-				} else if (time.Now().Unix() - startBelowLimit) < 30 {
-					// If the download is below the threshold for less than 30 seconds, continue
+				} else if (time.Now().Unix() - startBelowLimit) < slowTransferWindow {
+					// If the download is below the threshold for less than `SlowTransferWindow` (default 30) seconds, continue
 					continue
 				}
-				// The download is below the threshold for more than 30 seconds, cancel the download
+				// The download is below the threshold for more than `SlowTransferWindow` seconds, cancel the download
 				cancel()
 				if Options.ProgressBars {
 					var cancelledProgressBar = p.AddBar(0,

handle_http_test.go

@@ -5,19 +5,30 @@ import (
 	"net"
 	"net/http"
 	"net/http/httptest"
+	"net/http/httputil"
 	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 	"time"
+
 	"github.com/stretchr/testify/assert"
+	"github.com/spf13/viper"
 
 	"net/http/httputil"
 
 	namespaces "github.com/htcondor/osdf-client/v6/namespaces"
+	"github.com/htcondor/osdf-client/v6/config"
 )
 
+func TestMain(m *testing.M) {
+	if err := config.Init(); err != nil {
+		os.Exit(1)
+	}
+	os.Exit(m.Run())
+}
+
 // TestIsPort calls main.hasPort with a hostname, checking
 // for a valid return value.
 func TestIsPort(t *testing.T) {
@@ -93,6 +104,8 @@ func TestNewTransferDetailsEnv(t *testing.T) {
 	}
 
 	os.Setenv("OSG_DISABLE_PROXY_FALLBACK", "")
+	err := config.Init()
+	assert.Nil(t, err)
 	transfers := NewTransferDetails(testCache, false)
 	assert.Equal(t, 1, len(transfers))
 	assert.Equal(t, true, transfers[0].Proxy)
@@ -102,9 +115,16 @@ func TestNewTransferDetailsEnv(t *testing.T) {
 	assert.Equal(t, "https", transfers[0].Url.Scheme)
 	assert.Equal(t, false, transfers[0].Proxy)
 	os.Unsetenv("OSG_DISABLE_PROXY_FALLBACK")
+	viper.Reset()
+	err = config.Init()
+	assert.Nil(t, err)
 }
 
 func TestSlowTransfers(t *testing.T) {
+	// Adjust down some timeouts to speed up the test
+        viper.Set("SlowTransferWindow", 5)
+        viper.Set("SlowTransferRampupTime", 10)
+
 	channel := make(chan bool)
 	slowDownload := 1024 * 10 // 10 KiB/s < 100 KiB/s
 	svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -164,6 +184,10 @@ func TestSlowTransfers(t *testing.T) {
 
 // Test stopped transfer
 func TestStoppedTransfer(t *testing.T) {
+	// Adjust down the timeouts
+	viper.Set("StoppedTransferTimeout", 3)
+	viper.Set("SlowTransferRampupTime", 100)
+
 	channel := make(chan bool)
 	svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		buffer := make([]byte, 1024 * 100)

namespaces/namespaces_test.go

@@ -5,6 +5,9 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/spf13/viper"
+
+	"github.com/htcondor/osdf-client/v6/config"
 )
 
 // TestMatchNamespace calls MatchNamespace with a hostname, checking
@@ -68,6 +71,10 @@ func TestMatchNamespace(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
+	viper.Reset()
+	err = config.Init()
+	assert.Nil(t, err)
+
 	ns, err := MatchNamespace("/osgconnect/private/path/to/file.txt")
 	assert.NoError(t, err, "Failed to parse namespace")
 
@@ -150,6 +157,9 @@ func TestFullNamespace(t *testing.T) {
 // TestDownloadNamespaces tests the download of the namespaces JSON
 func TestDownloadNamespaces(t *testing.T) {
 	os.Setenv("STASH_NAMESPACE_URL", "https://topology-itb.opensciencegrid.org/stashcache/namespaces")
+	viper.Reset()
+	err := config.Init()
+	assert.Nil(t, err)
 	defer os.Unsetenv("STASH_NAMESPACE_URL")
 	namespaceBytes, err := downloadNamespace()
 	assert.NoError(t, err, "Failed to download namespaces")
@@ -159,6 +169,9 @@ func TestDownloadNamespaces(t *testing.T) {
 
 func TestDownloadNamespacesFail(t *testing.T) {
 	os.Setenv("STASH_NAMESPACE_URL", "https://doesnotexist.org.blah/namespaces.json")
+	viper.Reset()
+	err := config.Init()
+	assert.Nil(t, err)
 	defer os.Unsetenv("STASH_NAMESPACE_URL")
 	namespaceBytes, err := downloadNamespace()
 	assert.Error(t, err, "Failed to download namespaces")
@@ -168,6 +181,9 @@ func TestDownloadNamespacesFail(t *testing.T) {
 func TestGetNamespaces(t *testing.T) {
 	// Set the environment to an invalid URL, so it is forced to use the "built-in" namespaces.json
 	os.Setenv("STASH_NAMESPACE_URL", "https://doesnotexist.org.blah/namespaces.json")
+	viper.Reset()
+	err := config.Init()
+	assert.Nil(t, err)
 	defer os.Unsetenv("STASH_NAMESPACE_URL")
 	namespaces, err := GetNamespaces()
 	assert.NoError(t, err, "Failed to get namespaces")