Merge pull request #233 from cloudability/Region-URL-Support

Add CLOUDABILITY_UPLOAD_REGION Env Variable

Author: daniel-spray

README.md

@@ -58,9 +58,10 @@ Cloudability Metrics Agent currently does not support Rancher or On Prem cluster
 | CLOUDABILITY_LOG_LEVEL                         |                                                           Optional: Log level to run the agent at (INFO,WARN,DEBUG,TRACE). Default: `INFO`                                                           |
 | CLOUDABILITY_SCRATCH_DIR                       |  Optional: Temporary directory that metrics will be written to. If set, must assure that the directory exists and that the user agent UID 1000 has read/write access to the folder. Default: `/tmp`  |
 | CLOUDABILITY_NUMBER_OF_CONCURRENT_NODE_POLLERS |                                                   Optional: Number of goroutines that are created to poll node metrics in parallel. Default: `100`                                                   |
-| CLOUDABILITY_INFORMER_RESYNC_INTERVAL |                      Optional: Period of time (in hours) that the informers will fully resync the list of running resources. Default: 24 hours. Can be set to 0 to never resync                      |
+| CLOUDABILITY_INFORMER_RESYNC_INTERVAL          |                      Optional: Period of time (in hours) that the informers will fully resync the list of running resources. Default: 24 hours. Can be set to 0 to never resync                      |
 | CLOUDABILITY_PARSE_METRIC_DATA                 |                                        Optional: When true, core files will be parsed and non-relevant data will be removed prior to upload. Default: `false`                                        |
 | CLOUDABILITY_HTTPS_CLIENT_TIMEOUT              |                   Optional: Amount (in seconds) of time the http client has before timing out requests. Might need to be increased to clusters with large payloads. Default: `60`                    |
+| CLOUDABILITY_UPLOAD_REGION                     |                                            Optional: The region the metrics-agent will upload data to. Default `us-west-2`. Supported values: `us-west-2`                                            |
 
 ```sh
 

charts/metrics-agent/Chart.yaml

@@ -14,8 +14,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 2.11.15
+version: 2.11.16
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 2.11.15
+appVersion: 2.11.16

charts/metrics-agent/values.yaml

@@ -24,7 +24,7 @@ pollInterval: 180
 
 image:
   name: cloudability/metrics-agent
-  tag: 2.11.15
+  tag: 2.11.16
   pullPolicy: Always
 
 imagePullSecrets: []

client/client.go

@@ -30,7 +30,8 @@ import (
 
 //nolint gosec
 
-const defaultBaseURL = "https://metrics-collector.cloudability.com"
+const DefaultBaseURL string = "https://metrics-collector.cloudability.com/metricsample"
+const EUBaseURL string = "https://metrics-collector-eu.cloudability.com/metricsample"
 const defaultTimeout = 1 * time.Minute
 const defaultMaxRetries = 5
 
@@ -56,6 +57,7 @@ type Configuration struct {
 	ProxyAuth     string
 	ProxyInsecure bool
 	Verbose       bool
+	Region        string
 }
 
 // NewHTTPMetricClient will configure a new instance of a Cloudability client.
@@ -74,9 +76,9 @@ func NewHTTPMetricClient(cfg Configuration) (MetricClient, error) {
 	}
 	if len(strings.TrimSpace(cfg.BaseURL)) == 0 {
 		if cfg.Verbose {
-			log.Infof("Using default baseURL of %v", defaultBaseURL)
+			log.Infof("Using default baseURL of %v", DefaultBaseURL)
 		}
-		cfg.BaseURL = defaultBaseURL
+		cfg.BaseURL = GetUploadURLByRegion(cfg.Region)
 	}
 	if cfg.MaxRetries <= 0 {
 		if cfg.Verbose {
@@ -131,7 +133,6 @@ func NewHTTPMetricClient(cfg Configuration) (MetricClient, error) {
 
 // MetricClient represents a interface to send a cloudability measurement or metrics sample to an endpoint.
 type MetricClient interface {
-	SendMeasurement(measurements []measurement.Measurement) error
 	SendMetricSample(*os.File, string, string) error
 	GetUploadURL(*os.File, string, string, string, int) (string, string, error)
 }
@@ -150,56 +151,9 @@ type MetricSampleResponse struct {
 	Location string `json:"location"`
 }
 
-func (c httpMetricClient) SendMeasurement(measurements []measurement.Measurement) error {
-
-	measurementURL := c.baseURL + "/metrics"
-
-	b, err := toJSONLines(measurements)
-	if err != nil {
-		return err
-	}
-
-	req, err := http.NewRequest(http.MethodPost, measurementURL, bytes.NewBuffer(b))
-	if err != nil {
-		return err
-	}
-
-	req.Header.Set("Content-Type", "application/json")
-	req.Header.Set(authHeader, c.token)
-	req.Header.Set(apiKeyHeader, c.token)
-	req.Header.Set(userAgentHeader, c.userAgent)
-
-	if c.verbose {
-		requestDump, err := httputil.DumpRequest(req, true)
-		if err != nil {
-			log.Errorln(err)
-		}
-		log.Infoln(string(requestDump))
-	}
-
-	resp, err := c.httpClient.Do(req)
-	if err != nil {
-		return err
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("Request received %v response", resp.StatusCode)
-	}
-
-	if c.verbose {
-		responseDump, err := httputil.DumpResponse(resp, true)
-		if err != nil {
-			log.Errorln(err)
-		}
-		log.Infoln(string(responseDump))
-	}
-
-	return nil
-}
-
 // SendMetricSample uploads a file at a given path to the metrics endpoint.
 func (c httpMetricClient) SendMetricSample(metricSampleFile *os.File, agentVersion string, UID string) (rerr error) {
-	metricSampleURL := c.baseURL + "/metricsample"
+	metricSampleURL := c.baseURL
 
 	resp, err := c.retryWithBackoff(metricSampleURL, metricSampleFile, agentVersion, UID)
 	if err != nil {
@@ -452,3 +406,17 @@ func GetB64MD5Hash(name string) (b64Hash string, rerr error) {
 
 	return base64.StdEncoding.EncodeToString(h.Sum(nil)), err
 }
+
+// GetUploadURLByRegion returns the correct base url depending on the env variable CLOUDABILITY_UPLOAD_REGION.
+// If value is not supported, default to us-west-2 (original) URL
+func GetUploadURLByRegion(region string) string {
+	switch region {
+	case "eu-central-1":
+		return EUBaseURL
+	case "us-west-2":
+		return DefaultBaseURL
+	default:
+		log.Warnf("Region %s is not supported. Defaulting to us-west-2 region.", region)
+		return DefaultBaseURL
+	}
+}

client/client_test.go

@@ -5,7 +5,6 @@ import (
 	"bytes"
 	"compress/gzip"
 	"encoding/json"
-	"io"
 	"math/rand"
 	"net"
 	"net/http"
@@ -18,9 +17,10 @@ import (
 	"github.com/cloudability/metrics-agent/client"
 	"github.com/cloudability/metrics-agent/measurement"
 	"github.com/cloudability/metrics-agent/test"
-	"github.com/cloudability/metrics-agent/version"
 )
 
+const metricsSuffix string = "/metricsample"
+
 // nolint: dupl
 func TestClientCreation(t *testing.T) {
 	t.Parallel()
@@ -31,102 +31,14 @@ func TestClientCreation(t *testing.T) {
 			Token:      test.SecureRandomAlphaString(20),
 			BaseURL:    "https://cloudability.com",
 			MaxRetries: 2,
+			Region:     "us-west-2",
 		})
 		if c == nil || err != nil {
 			t.Error("Expected client to successfully create")
 		}
 	})
 }
 
-func TestSendMeasurement(t *testing.T) {
-	t.Parallel()
-
-	token := test.SecureRandomAlphaString(20)
-
-	tags := make(map[string]string)
-	tags["host"] = "macbookpro.Local.abc123"
-	tags["cat"] = "dog"
-
-	measure := measurement.Measurement{
-		Name:      "ametric",
-		Value:     1243.00,
-		Timestamp: time.Now().Unix(),
-		Tags:      tags,
-	}
-	m := []measurement.Measurement{measure}
-	jsonBytes, _ := client.ToJSONLines(m)
-
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.Method != http.MethodPost {
-			t.Error("Expected to be a POST")
-		}
-		if !strings.Contains(r.UserAgent(), version.VERSION) {
-			t.Error("Expected version to be present in user agent")
-		}
-		if r.Header.Get(client.AuthHeader) != token {
-			t.Error("Expected token to be set on request")
-		}
-		if r.Header.Get(client.APIKeyHeader) != token {
-			t.Error("Expected token to be set on request")
-		}
-		bodyBytes, _ := io.ReadAll(r.Body)
-		if !bytes.Equal(jsonBytes, bodyBytes) {
-			t.Errorf("Did not receive expect json.  Got %v, expected %v", string(bodyBytes), string(jsonBytes))
-		}
-		w.WriteHeader(200)
-	}))
-	defer ts.Close()
-
-	c, err := client.NewHTTPMetricClient(client.Configuration{
-		Timeout:    10 * time.Second,
-		Token:      token,
-		BaseURL:    ts.URL,
-		MaxRetries: 2,
-	})
-	if err != nil {
-		t.Error(err)
-	}
-
-	err = c.SendMeasurement(m)
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func TestSendMeasurement_ErrorState(t *testing.T) {
-	t.Parallel()
-
-	t.Run("404 Not Found", func(t *testing.T) {
-		t.Parallel()
-
-		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			w.WriteHeader(404)
-		}))
-		//nolint errcheck
-		defer ts.Close()
-
-		c, err := client.NewHTTPMetricClient(client.Configuration{
-			Timeout:    10 * time.Second,
-			Token:      test.SecureRandomAlphaString(20),
-			BaseURL:    ts.URL,
-			MaxRetries: 2,
-		})
-		if err != nil {
-			t.Error(err)
-		}
-		m := []measurement.Measurement{{}}
-
-		err = c.SendMeasurement(m)
-		if err == nil {
-			t.Errorf("Expected to receive an error")
-		}
-		if !strings.Contains(err.Error(), "404") {
-			t.Logf("Error message: %v", err.Error())
-			t.Errorf("Expected 404 status code in response")
-		}
-	})
-}
-
 func TestToJSONLines(t *testing.T) {
 	t.Parallel()
 
@@ -256,7 +168,8 @@ func TestSendMetricSample(t *testing.T) {
 		Timeout:    10 * time.Second,
 		Token:      token,
 		MaxRetries: 2,
-		BaseURL:    ts.URL,
+		BaseURL:    ts.URL + metricsSuffix,
+		Region:     "us-west-2",
 	})
 	if err != nil {
 		t.Error(err)
@@ -298,6 +211,7 @@ func TestSendMetricSample_ErrorState(t *testing.T) {
 			Token:      test.SecureRandomAlphaString(20),
 			BaseURL:    ts.URL,
 			MaxRetries: 2,
+			Region:     "us-west-2",
 		})
 		if err != nil {
 			t.Error(err)
@@ -344,8 +258,9 @@ func TestSendMetricSample_ErrorState(t *testing.T) {
 		c, err := client.NewHTTPMetricClient(client.Configuration{
 			Timeout:    10 * time.Second,
 			Token:      test.SecureRandomAlphaString(20),
-			BaseURL:    ts.URL,
+			BaseURL:    ts.URL + metricsSuffix,
 			MaxRetries: 2,
+			Region:     "us-west-2",
 		})
 		if err != nil {
 			t.Error(err)
@@ -399,6 +314,7 @@ func TestSendMetricSample_ErrorState(t *testing.T) {
 			Token:      test.SecureRandomAlphaString(20),
 			MaxRetries: 2,
 			BaseURL:    ts.URL,
+			Region:     "us-west-2",
 		})
 		if err != nil {
 			t.Error(err)
@@ -449,6 +365,7 @@ func TestSendMetricSample_ErrorState(t *testing.T) {
 			Token:      test.SecureRandomAlphaString(20),
 			MaxRetries: 2,
 			BaseURL:    ts.URL,
+			Region:     "us-west-2",
 		})
 		if err != nil {
 			t.Error(err)
@@ -500,8 +417,9 @@ func TestSendMetricSample_ErrorState(t *testing.T) {
 		c, err := client.NewHTTPMetricClient(client.Configuration{
 			Timeout:    10 * time.Second,
 			Token:      test.SecureRandomAlphaString(20),
-			BaseURL:    ts.URL,
+			BaseURL:    ts.URL + metricsSuffix,
 			MaxRetries: 2,
+			Region:     "us-west-2",
 		})
 		if err != nil {
 			t.Error(err)
@@ -584,6 +502,7 @@ func TestGetUploadURL(t *testing.T) {
 		Token:      token,
 		MaxRetries: 2,
 		BaseURL:    ts.URL,
+		Region:     "us-west-2",
 	})
 	if err != nil {
 		t.Error(err)
@@ -614,3 +533,21 @@ func Test_getB64MD5Hash(t *testing.T) {
 	}
 
 }
+
+func Test_getUploadURLByRegion(t *testing.T) {
+	// us-west-2 url generation
+	uploadURL := client.GetUploadURLByRegion("us-west-2")
+	if uploadURL != client.DefaultBaseURL {
+		t.Error("US URL was not generated correctly")
+	}
+	// eu-central-1 url generation
+	uploadURL = client.GetUploadURLByRegion("eu-central-1")
+	if uploadURL != client.EUBaseURL {
+		t.Error("EU URL was not generated correctly")
+	}
+	// unsupported region should default to us-west-2 url generation
+	uploadURL = client.GetUploadURLByRegion("my-unsupported-region-1")
+	if uploadURL != client.DefaultBaseURL {
+		t.Error("Unsupported region default to US URL was not generated correctly")
+	}
+}

cmd/kubernetesCmd.go

@@ -132,6 +132,12 @@ func init() {
 		60,
 		"Amount (in seconds) of time the https client has before timing out requests. Default 60",
 	)
+	kubernetesCmd.PersistentFlags().StringVar(
+		&config.UploadRegion,
+		"upload_region",
+		"us-west-2",
+		"The region the metrics-agent will upload data to. Default us-west-2",
+	)
 
 	//nolint gas
 	_ = viper.BindPFlag("api_key", kubernetesCmd.PersistentFlags().Lookup("api_key"))
@@ -156,6 +162,7 @@ func init() {
 		kubernetesCmd.PersistentFlags().Lookup("number_of_concurrent_node_pollers"))
 	_ = viper.BindPFlag("parse_metric_data", kubernetesCmd.PersistentFlags().Lookup("parse_metric_data"))
 	_ = viper.BindPFlag("https_client_timeout", kubernetesCmd.PersistentFlags().Lookup("https_client_timeout"))
+	_ = viper.BindPFlag("upload_region", kubernetesCmd.PersistentFlags().Lookup("upload_region"))
 
 	viper.SetEnvPrefix("cloudability")
 	viper.AutomaticEnv()
@@ -180,6 +187,7 @@ func init() {
 		InformerResyncInterval: viper.GetInt("informer_resync_interval"),
 		ParseMetricData:        viper.GetBool("parse_metric_data"),
 		HTTPSTimeout:           viper.GetInt("https_client_timeout"),
+		UploadRegion:           viper.GetString("upload_region"),
 	}
 
 }