feat: support a proxy for the DOI backend - Rclone 1.65.2

backend/doi/doi.go

@@ -73,6 +73,16 @@ The DOI provider can be set when rclone does not automatically recognize a suppo
 				}},
 			Required: false,
 			Advanced: true,
+		}, {
+			Name:     "proxy_url",
+			Help:     "Proxy URL",
+			Required: false,
+			Advanced: true,
+		}, {
+			Name:     "proxy_ca_cert",
+			Help:     "Proxy CA certificate",
+			Required: false,
+			Advanced: true,
 		}},
 	}
 	fs.Register(fsi)
@@ -81,7 +91,7 @@ The DOI provider can be set when rclone does not automatically recognize a suppo
 // Provider defines the type of provider hosting the DOI
 type Provider string
 
-var (
+const (
 	// Zenodo provider, see https://zenodo.org
 	Zenodo Provider = "zenodo"
 	// Dataverse provider, see https://dataverse.harvard.edu
@@ -92,8 +102,10 @@ var (
 
 // Options defines the configuration for this backend
 type Options struct {
-	Doi      string `config:"doi"`      // The DOI, a digital identifier of an object, usually a dataset
-	Provider string `config:"provider"` // The DOI provider
+	Doi         string `config:"doi"`      // The DOI, a digital identifier of an object, usually a dataset
+	Provider    string `config:"provider"` // The DOI provider
+	ProxyURL    string `config:"proxy_url"`
+	ProxyCACert string `config:"proxy_ca_cert"`
 }
 
 // Fs stores the interface to the remote HTTP files
@@ -133,7 +145,7 @@ func parseDoi(doi string) string {
 		return doi
 	}
 	if doiURL.Scheme == "doi" {
-		return strings.TrimLeft(strings.TrimLeft(doi, "doi:"), "/")
+		return strings.TrimLeft(strings.TrimPrefix(doi, "doi:"), "/")
 	}
 	if strings.HasSuffix(doiURL.Hostname(), "doi.org") {
 		return strings.TrimLeft(doiURL.Path, "/")
@@ -278,7 +290,16 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}
 	opt.Doi = parseDoi(opt.Doi)
 
-	client := fshttp.NewClient(ctx)
+	var client *http.Client
+	if opt.ProxyURL != "" {
+		client, err = newHttpClient(ctx, opt)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		client = fshttp.NewClient(ctx)
+	}
+
 	ci := fs.GetConfig(ctx)
 	f := &Fs{
 		name:  name,

backend/doi/invenio.go

@@ -159,7 +159,7 @@ func (f *Fs) listInvevioDoiFiles(ctx context.Context) (entries []*Object, err er
 			size:        file.Size,
 			modTime:     modTime,
 			contentType: file.MimeType,
-			md5:         strings.TrimLeft(file.Checksum, "md5:"),
+			md5:         strings.TrimPrefix(file.Checksum, "md5:"),
 		}
 		entries = append(entries, entry)
 	}

backend/doi/proxy.go

@@ -0,0 +1,64 @@
+// Utilities to use a caching proxy
+
+package doi
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fshttp"
+)
+
+func newHttpClient(ctx context.Context, opt *Options) (client *http.Client, err error) {
+	client = fshttp.NewClient(ctx)
+
+	proxyUrl, err := url.Parse(opt.ProxyURL)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get the SystemCertPool, continue with an empty pool on error
+	rootCAs, _ := x509.SystemCertPool()
+	if rootCAs == nil {
+		rootCAs = x509.NewCertPool()
+	}
+
+	if opt.ProxyCACert != "" {
+		fs.Logf(nil, "Adding to root CAs: %s", opt.ProxyCACert)
+		// Read in the self-signed certificate
+		certs, err := os.ReadFile(opt.ProxyCACert)
+		if err != nil {
+			return nil, err
+		}
+		// Add the certificate to the root CAs
+		if ok := rootCAs.AppendCertsFromPEM(certs); !ok {
+			return nil, fmt.Errorf("could not append self-signed certificate to the CA pool")
+		}
+	}
+
+	defaultTransport := http.DefaultTransport.(*http.Transport)
+	transport := http.Transport{
+		Proxy: func(r *http.Request) (*url.URL, error) {
+			return proxyUrl, nil
+		},
+		DialContext:           defaultTransport.DialContext,
+		ForceAttemptHTTP2:     defaultTransport.ForceAttemptHTTP2,
+		MaxIdleConns:          defaultTransport.MaxIdleConns,
+		IdleConnTimeout:       defaultTransport.IdleConnTimeout,
+		TLSHandshakeTimeout:   defaultTransport.TLSHandshakeTimeout,
+		ExpectContinueTimeout: defaultTransport.ExpectContinueTimeout,
+	}
+	if transport.TLSClientConfig == nil {
+		transport.TLSClientConfig = &tls.Config{}
+	}
+	transport.TLSClientConfig.RootCAs = rootCAs
+	client.Transport = &transport
+
+	return client, nil
+}