Merge pull request #3092 from cozy/network

Improve http client options

Author: nono

model/app/fetcher_http.go

@@ -8,6 +8,7 @@ import (
 	"encoding/hex"
 	"hash"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
@@ -22,7 +23,7 @@ import (
 )
 
 var httpClient = http.Client{
-	Timeout: 2 * 60 * time.Second,
+	Timeout: 60 * time.Second,
 }
 
 type httpFetcher struct {
@@ -49,6 +50,8 @@ func (f *httpFetcher) FetchManifest(src *url.URL) (r io.ReadCloser, err error) {
 	}
 	defer func() {
 		if err != nil {
+			// Flush the body, so that the connecion can be reused by keep-alive
+			_, _ = io.Copy(ioutil.Discard, resp.Body)
 			resp.Body.Close()
 		}
 	}()
@@ -125,6 +128,8 @@ func fetchHTTP(src *url.URL, shasum []byte, fs appfs.Copier, man Manifest, prefi
 	resp, err := httpClient.Do(req)
 	elapsed := time.Since(start)
 	if err != nil {
+		log := logger.WithNamespace("fetcher")
+		log.Printf("cannot fetch %s: %s", src.String(), err)
 		return err
 	}
 	defer resp.Body.Close()

model/bitwarden/icon.go

@@ -23,6 +23,11 @@ const (
 
 var iconClient = &http.Client{
 	Timeout: 10 * time.Second,
+	// As we are connecting to a new host each time, it is better to disable
+	// keep-alive
+	Transport: &http.Transport{
+		DisableKeepAlives: true,
+	},
 }
 
 // Icon is a simple struct with a content-type and the content of an icon.

pkg/config/config/config.go

@@ -490,8 +490,9 @@ func UseViper(v *viper.Viper) error {
 		couchURL.Path = "/"
 	}
 	couchClient, _, err := tlsclient.NewHTTPClient(tlsclient.HTTPEndpoint{
-		Timeout:    10 * time.Second,
-		RootCAFile: v.GetString("couchdb.root_ca"),
+		Timeout:             10 * time.Second,
+		MaxIdleConnsPerHost: 20,
+		RootCAFile:          v.GetString("couchdb.root_ca"),
 		ClientCertificateFiles: tlsclient.ClientCertificateFilePair{
 			CertificateFile: v.GetString("couchdb.client_cert"),
 			KeyFile:         v.GetString("couchdb.client_key"),

pkg/couchdb/couchdb.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/url"
@@ -344,6 +345,8 @@ func makeRequest(db Database, doctype, method, path string, reqbody interface{},
 		return err
 	}
 	if resbody == nil {
+		// Flush the body, so that the connecion can be reused by keep-alive
+		_, _ = io.Copy(ioutil.Discard, resp.Body)
 		return nil
 	}
 

pkg/manager/client.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"io"
 	"net/http"
+	"time"
 
 	"golang.org/x/oauth2"
 )
@@ -33,9 +34,11 @@ type APIClient struct {
 // NewAPIClient builds a new client for the manager API
 func NewAPIClient(baseURL, token string) *APIClient {
 	tokenSource := &tokenSource{token: token}
+	client := oauth2.NewClient(context.Background(), tokenSource)
+	client.Timeout = 15 * time.Second
 	return &APIClient{
 		baseURL: baseURL,
-		client:  oauth2.NewClient(context.Background(), tokenSource),
+		client:  client,
 	}
 }
 

pkg/registry/registry.go

@@ -4,6 +4,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"net/url"
@@ -502,6 +504,8 @@ func fetch(client *http.Client, registry, ref *url.URL, cache CacheControl) (res
 	elapsed := time.Since(start)
 	defer func() {
 		if !ok {
+			// Flush the body, so that the connecion can be reused by keep-alive
+			_, _ = io.Copy(ioutil.Discard, resp.Body)
 			resp.Body.Close()
 		}
 	}()

web/oidc/oidc.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/url"
@@ -357,6 +358,8 @@ func getToken(conf *Config, code string) (string, error) {
 	}
 	defer res.Body.Close()
 	if res.StatusCode != 200 {
+		// Flush the body, so that the connecion can be reused by keep-alive
+		_, _ = io.Copy(ioutil.Discard, res.Body)
 		logger.WithNamespace("oidc").
 			Infof("Invalid status code %d for %s", res.StatusCode, conf.TokenURL)
 		return "", fmt.Errorf("OIDC service responded with %d", res.StatusCode)
@@ -426,6 +429,8 @@ func getUserInfo(conf *Config, token string) (map[string]interface{}, error) {
 	}
 	defer res.Body.Close()
 	if res.StatusCode != 200 {
+		// Flush the body, so that the connecion can be reused by keep-alive
+		_, _ = io.Copy(ioutil.Discard, res.Body)
 		logger.WithNamespace("oidc").
 			Infof("Invalid status code %d for %s", res.StatusCode, conf.UserInfoURL)
 		return nil, fmt.Errorf("OIDC service responded with %d", res.StatusCode)