Silently ignore un-parsable links such as data: when crawling.

PiperOrigin-RevId: 871251237

Author: tooryx

common/clients/httpcrawler/parser/parser.go

@@ -33,11 +33,15 @@ import (
 var (
 	// ErrParseURL is returned when the URL fails to parse.
 	ErrParseURL = errors.New("failed to parse URL")
-	// ErrUnsupportedURLType is returned when the URL is a javascript or mailto URL.
-	ErrUnsupportedURLType = errors.New("unsupported javascript/mailto URL type")
 	// ErrUnsupportedScheme is returned when the URL scheme is not supported.
 	ErrUnsupportedScheme = errors.New("unsupported scheme")
 
+	unsupportedPrefixes = []string{
+		"data:",
+		"javascript:",
+		"mailto:",
+	}
+
 	knownLinkAttributes = []string{
 		// HTML 4 link attributes.
 		"action",
@@ -109,8 +113,10 @@ func processHTMLNode(rootURL string, node *html.Node) ([]string, error) {
 }
 
 func parseURL(base string, redirect string) (string, error) {
-	if strings.HasPrefix(redirect, "javascript:") || strings.HasPrefix(redirect, "mailto:") {
-		return "", ErrUnsupportedURLType
+	for _, prefix := range unsupportedPrefixes {
+		if strings.HasPrefix(redirect, prefix) {
+			return "", nil
+		}
 	}
 
 	redirurl, err := url.Parse(redirect)

common/clients/httpcrawler/parser/parser_test.go

@@ -229,16 +229,25 @@ func TestParseURL(t *testing.T) {
 			wantErr: ErrUnsupportedScheme,
 		},
 		{
-			name:    "when_node_is_javascript_returns_error",
+			name:    "when_node_is_javascript_returns_nothing",
 			rootURL: "http://domain.com/",
-			nodeURL: "javascript:alert('Evil XSS')",
-			wantErr: ErrUnsupportedURLType,
+			nodeURL: "javascript:alert('hello')",
+			wantErr: nil,
+			want:    "",
 		},
 		{
-			name:    "when_node_is_mailto_returns_error",
+			name:    "when_node_is_mailto_returns_nothing",
 			rootURL: "http://domain.com/",
 			nodeURL: "mailto:someone@domain.com",
-			wantErr: ErrUnsupportedURLType,
+			wantErr: nil,
+			want:    "",
+		},
+		{
+			name:    "when_node_is_data_url_returns_nothing",
+			rootURL: "http://domain.com/",
+			nodeURL: "data:text/plain;base64,SGVsbG8sIFdvcmxkIQ==",
+			wantErr: nil,
+			want:    "",
 		},
 	}