web: allow overriding referer and user-agent

Author: chapmanjacobd

library/utils/arggroups.py

@@ -1685,6 +1685,10 @@ def requests(parent_parser):
     parser.add_argument(
         "--http-retries", "--http-max-retries", "--retries", type=int, default=8, help="Use N retries for requests"
     )
+    parser.add_argument(
+        "--referrer", "--http-referrer", "--referer", "--http-referer", help="Use a custom Referer header"
+    )
+    parser.add_argument("--user-agent", "--http-user-agent", help="Use a custom User-Agent header")
     parser.add_argument(
         "--http-max-redirects",
         "--max-redirects",

library/utils/web.py

@@ -89,10 +89,14 @@ def parse_cookies_from_browser(input_str):
 def requests_session(args=argparse.Namespace()):
     global session
 
-    from yt_dlp.utils.networking import std_headers
-
     if session is None:
         import requests
+        from yt_dlp.utils.networking import std_headers
+
+        if getattr(args, "referrer"):
+            std_headers |= {"Referer": args.referrer}
+        if getattr(args, "user_agent"):
+            std_headers |= {"User-Agent": args.user_agent}
 
         cookie_file = getattr(args, "cookies", None)
         cookies_from_browser = getattr(args, "cookies_from_browser", None)