Merge pull request #876 from cgwalters/fetch-headers-prep

Rework fetch/resource to automatically append required headers

Author: Andrew Jeddeloh

internal/exec/engine.go

@@ -272,9 +272,7 @@ func (e *Engine) fetchReferencedConfig(cfgRef types.ConfigReference) (types.Conf
 	if err != nil {
 		return types.Config{}, err
 	}
-	rawCfg, err := e.Fetcher.FetchToBuffer(*u, resource.FetchOptions{
-		Headers: resource.ConfigHeaders,
-	})
+	rawCfg, err := e.Fetcher.FetchToBuffer(*u, resource.FetchOptions{})
 	if err != nil {
 		return types.Config{}, err
 	}

internal/providers/aliyun/aliyun.go

@@ -36,9 +36,7 @@ var (
 )
 
 func FetchConfig(f *resource.Fetcher) (types.Config, report.Report, error) {
-	data, err := f.FetchToBuffer(userdataUrl, resource.FetchOptions{
-		Headers: resource.ConfigHeaders,
-	})
+	data, err := f.FetchToBuffer(userdataUrl, resource.FetchOptions{})
 	if err != nil && err != resource.ErrNotFound {
 		return types.Config{}, report.Report{}, err
 	}

internal/providers/aws/aws.go

@@ -41,9 +41,7 @@ var (
 )
 
 func FetchConfig(f *resource.Fetcher) (types.Config, report.Report, error) {
-	data, err := f.FetchToBuffer(userdataUrl, resource.FetchOptions{
-		Headers: resource.ConfigHeaders,
-	})
+	data, err := f.FetchToBuffer(userdataUrl, resource.FetchOptions{})
 	if err != nil && err != resource.ErrNotFound {
 		return types.Config{}, report.Report{}, err
 	}

internal/providers/cloudstack/cloudstack.go

@@ -204,8 +204,6 @@ func fetchConfigFromMetadataService(f *resource.Fetcher) ([]byte, error) {
 		Path:   "/latest/user-data",
 	}
 
-	res, err := f.FetchToBuffer(metadataServiceUrl, resource.FetchOptions{
-		Headers: resource.ConfigHeaders,
-	})
+	res, err := f.FetchToBuffer(metadataServiceUrl, resource.FetchOptions{})
 	return res, err
 }

internal/providers/cmdline/cmdline.go

@@ -46,9 +46,7 @@ func FetchConfig(f *resource.Fetcher) (types.Config, report.Report, error) {
 		return types.Config{}, report.Report{}, providers.ErrNoProvider
 	}
 
-	data, err := f.FetchToBuffer(*url, resource.FetchOptions{
-		Headers: resource.ConfigHeaders,
-	})
+	data, err := f.FetchToBuffer(*url, resource.FetchOptions{})
 	if err != nil {
 		return types.Config{}, report.Report{}, err
 	}

internal/providers/digitalocean/digitalocean.go

@@ -36,9 +36,7 @@ var (
 )
 
 func FetchConfig(f *resource.Fetcher) (types.Config, report.Report, error) {
-	data, err := f.FetchToBuffer(userdataUrl, resource.FetchOptions{
-		Headers: resource.ConfigHeaders,
-	})
+	data, err := f.FetchToBuffer(userdataUrl, resource.FetchOptions{})
 	if err != nil {
 		return types.Config{}, report.Report{}, err
 	}

internal/providers/gcp/gcp.go

@@ -18,6 +18,7 @@
 package gcp
 
 import (
+	"net/http"
 	"net/url"
 
 	"github.com/coreos/ignition/v2/config/v3_1_experimental/types"
@@ -38,7 +39,7 @@ var (
 )
 
 func FetchConfig(f *resource.Fetcher) (types.Config, report.Report, error) {
-	headers := resource.ConfigHeaders
+	headers := make(http.Header)
 	headers.Set(metadataHeaderKey, metadataHeaderVal)
 	data, err := f.FetchToBuffer(userdataUrl, resource.FetchOptions{
 		Headers: headers,

internal/providers/openstack/openstack.go

@@ -131,8 +131,6 @@ func fetchConfigFromDevice(logger *log.Logger, ctx context.Context, path string)
 }
 
 func fetchConfigFromMetadataService(f *resource.Fetcher) ([]byte, error) {
-	res, err := f.FetchToBuffer(metadataServiceUrl, resource.FetchOptions{
-		Headers: resource.ConfigHeaders,
-	})
+	res, err := f.FetchToBuffer(metadataServiceUrl, resource.FetchOptions{})
 	return res, err
 }

internal/providers/packet/packet.go

@@ -55,7 +55,7 @@ var (
 func FetchConfig(f *resource.Fetcher) (types.Config, report.Report, error) {
 	// Packet's metadata service returns "Not Acceptable" when queried
 	// with the default Accept header.
-	headers := resource.ConfigHeaders
+	headers := make(http.Header)
 	headers.Set("Accept", "*/*")
 	data, err := f.FetchToBuffer(userdataUrl, resource.FetchOptions{
 		Headers: headers,

internal/resource/url.go

@@ -52,7 +52,7 @@ var (
 
 	// ConfigHeaders are the HTTP headers that should be used when the Ignition
 	// config is being fetched
-	ConfigHeaders = http.Header{
+	configHeaders = http.Header{
 		"Accept-Encoding": []string{"identity"},
 		"Accept":          []string{"application/vnd.coreos.ignition+json;version=3.0.0, */*;q=0.1"},
 	}
@@ -248,7 +248,21 @@ func (f *Fetcher) fetchFromHTTP(u url.URL, dest io.Writer, opts FetchOptions) er
 		}
 	}
 
-	dataReader, status, ctxCancel, err := f.client.getReaderWithHeader(u.String(), opts.Headers)
+	// TODO use .Clone() when we have a new enough golang
+	// (With Rust, we'd have immutability and wouldn't need to defensively clone)
+	headers := make(http.Header)
+	for k, va := range opts.Headers {
+		for _, v := range va {
+			headers.Add(k, v)
+		}
+	}
+	for k, va := range configHeaders {
+		for _, v := range va {
+			headers.Add(k, v)
+		}
+	}
+
+	dataReader, status, ctxCancel, err := f.client.getReaderWithHeader(u.String(), headers)
 	if ctxCancel != nil {
 		// whatever context getReaderWithHeader created for the request should
 		// be cancelled once we're done reading the response