diff --git a/backend/doi/doi.go b/backend/doi/doi.go index c5fa281603f1a..886a9fa2a1114 100644 --- a/backend/doi/doi.go +++ b/backend/doi/doi.go @@ -73,6 +73,16 @@ The DOI provider can be set when rclone does not automatically recognize a suppo }}, Required: false, Advanced: true, + }, { + Name: "proxy_url", + Help: "Proxy URL", + Required: false, + Advanced: true, + }, { + Name: "proxy_ca_cert", + Help: "Proxy CA certificate", + Required: false, + Advanced: true, }}, } fs.Register(fsi) @@ -81,7 +91,7 @@ The DOI provider can be set when rclone does not automatically recognize a suppo // Provider defines the type of provider hosting the DOI type Provider string -var ( +const ( // Zenodo provider, see https://zenodo.org Zenodo Provider = "zenodo" // Dataverse provider, see https://dataverse.harvard.edu @@ -92,8 +102,10 @@ var ( // Options defines the configuration for this backend type Options struct { - Doi string `config:"doi"` // The DOI, a digital identifier of an object, usually a dataset - Provider string `config:"provider"` // The DOI provider + Doi string `config:"doi"` // The DOI, a digital identifier of an object, usually a dataset + Provider string `config:"provider"` // The DOI provider + ProxyURL string `config:"proxy_url"` + ProxyCACert string `config:"proxy_ca_cert"` } // Fs stores the interface to the remote HTTP files @@ -133,7 +145,7 @@ func parseDoi(doi string) string { return doi } if doiURL.Scheme == "doi" { - return strings.TrimLeft(strings.TrimLeft(doi, "doi:"), "/") + return strings.TrimLeft(strings.TrimPrefix(doi, "doi:"), "/") } if strings.HasSuffix(doiURL.Hostname(), "doi.org") { return strings.TrimLeft(doiURL.Path, "/") @@ -278,7 +290,16 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e } opt.Doi = parseDoi(opt.Doi) - client := fshttp.NewClient(ctx) + var client *http.Client + if opt.ProxyURL != "" { + client, err = newHttpClient(ctx, opt) + if err != nil { + return nil, err + } + } else { + client = fshttp.NewClient(ctx) + } + ci := fs.GetConfig(ctx) f := &Fs{ name: name, diff --git a/backend/doi/invenio.go b/backend/doi/invenio.go index 76659db60d915..fe318f64ae61e 100644 --- a/backend/doi/invenio.go +++ b/backend/doi/invenio.go @@ -159,7 +159,7 @@ func (f *Fs) listInvevioDoiFiles(ctx context.Context) (entries []*Object, err er size: file.Size, modTime: modTime, contentType: file.MimeType, - md5: strings.TrimLeft(file.Checksum, "md5:"), + md5: strings.TrimPrefix(file.Checksum, "md5:"), } entries = append(entries, entry) } diff --git a/backend/doi/proxy.go b/backend/doi/proxy.go new file mode 100644 index 0000000000000..de30c834292c2 --- /dev/null +++ b/backend/doi/proxy.go @@ -0,0 +1,64 @@ +// Utilities to use a caching proxy + +package doi + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "net/http" + "net/url" + "os" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/fshttp" +) + +func newHttpClient(ctx context.Context, opt *Options) (client *http.Client, err error) { + client = fshttp.NewClient(ctx) + + proxyUrl, err := url.Parse(opt.ProxyURL) + if err != nil { + return nil, err + } + + // Get the SystemCertPool, continue with an empty pool on error + rootCAs, _ := x509.SystemCertPool() + if rootCAs == nil { + rootCAs = x509.NewCertPool() + } + + if opt.ProxyCACert != "" { + fs.Logf(nil, "Adding to root CAs: %s", opt.ProxyCACert) + // Read in the self-signed certificate + certs, err := os.ReadFile(opt.ProxyCACert) + if err != nil { + return nil, err + } + // Add the certificate to the root CAs + if ok := rootCAs.AppendCertsFromPEM(certs); !ok { + return nil, fmt.Errorf("could not append self-signed certificate to the CA pool") + } + } + + defaultTransport := http.DefaultTransport.(*http.Transport) + transport := http.Transport{ + Proxy: func(r *http.Request) (*url.URL, error) { + return proxyUrl, nil + }, + DialContext: defaultTransport.DialContext, + ForceAttemptHTTP2: defaultTransport.ForceAttemptHTTP2, + MaxIdleConns: defaultTransport.MaxIdleConns, + IdleConnTimeout: defaultTransport.IdleConnTimeout, + TLSHandshakeTimeout: defaultTransport.TLSHandshakeTimeout, + ExpectContinueTimeout: defaultTransport.ExpectContinueTimeout, + } + if transport.TLSClientConfig == nil { + transport.TLSClientConfig = &tls.Config{} + } + transport.TLSClientConfig.RootCAs = rootCAs + client.Transport = &transport + + return client, nil +}