Merge pull request #10848 from cdrini/perf/add-more-nginx-rules

Add nginx blocks for older browsers + rejigger nginx conf

Author: mekarpeles

docker/nginx.conf

@@ -74,12 +74,6 @@ http {
     # Set a more permissive limit for covers because some pages might load 20+ covers.
     limit_req_zone $rate_limit_key zone=cover_limit:10m rate=400r/m;
 
-    # For returning 200 when someone tries to randomly sort author results.
-    map $arg_sort $is_random_sort {
-      default 0;
-      ~^random_.* 1;
-    }
-
     # Things are mounted into here by the docker compose file
     include /etc/nginx/sites-enabled/*;
 }

docker/web_nginx.conf

@@ -16,6 +16,38 @@ upstream webnodes {
     server web_haproxy:7072;
 }
 
+map $http_user_agent $is_sus_user_agent {
+    default 0;
+    "~ByteSpider" 1;
+    # Check for ancient browser user agents
+    "~Firefox/([1-7]\.)" 1;
+    "~Chrome/([1-9]|10)\." 1;
+}
+
+# Log likely bots caught in /authors random loop.
+map $request_uri$args $is_sus_random_sort {
+    default 0;
+    "~^/authors/.*sort=random_" 1;
+}
+
+# Check empty referer
+map $http_referer $is_empty_referer {
+    default 0;
+    "" 1;
+}
+
+map $request_uri$args $requires_referer {
+    default 0;
+    "~*^/(qrcode|admin|wp-login|show-records)" 1;
+    "~*/edit" 1;
+    "~*(v|m|action|redirect)=" 1;
+}
+
+map $is_empty_referer$requires_referer $is_sus_referer {
+    default 0;
+    "11" 1;
+}
+
 # Keep in sync with covers_nginx.conf
 server {
     listen 80 default;
@@ -85,38 +117,19 @@ server {
         limit_req zone=web_limit burst=100 delay=10;
         limit_req_status 429;
 
-        if ($http_user_agent ~ (Bytespider) ) {
-           return 444;
-        }
-
-
-        # ===========================================
-        # Block certain patterns when no referer set:
-        # ===========================================
-
-        # Create a variable to track if referer is empty
-        set $suspect_arg 0;
-
-        # These requests should not be hit without referrer
-        if ($request_uri ~* "/(qrcode|admin|wp-login|show-records|edit)") {
-            set $suspect_arg 1;
-        }
-        if ($args ~* "(v=|m=|action=)") {
-            set $suspect_arg 1;
+        # For returning 200 when someone tries to randomly sort author results.
+        if ($is_sus_random_sort) {
+            return 200;
         }
 
-        # AND if the referer is set...
-        if ($http_referer = "" ) {
-            set $suspect_arg "${suspect_arg}1";
+        if ($is_sus_user_agent) {
+            return 403;
         }
 
-        # Block requests with m= v= or action= parameters and empty referer
-        if ($suspect_arg = "11") {
+        if ($is_sus_referer) {
             return 444;
         }
 
-        # -------------------------------------------
-
         proxy_pass http://webnodes;
         proxy_set_header Host $http_host;
 
@@ -143,27 +156,6 @@ server {
         proxy_set_header X-Scheme $scheme;
     }
 
-    # Log likely bots caught in /authors random loop.
-    location ~* ^/authors/.* {
-        # Web rate limit.
-        limit_req zone=web_limit burst=100 delay=10;
-        limit_req_status 429;
-
-        # randomly sorting will be removed. For now just return 200
-        if ($is_random_sort = 1) {
-            return 200 "";
-        }
-
-        if ($http_user_agent ~ (Bytespider) ) {
-            return 444;
-        }
-
-        proxy_pass http://webnodes;
-        proxy_set_header Host $http_host;
-        proxy_set_header X-Forwarded-For $remote_addr;
-        proxy_set_header X-Scheme $scheme;
-    }
-
     location ^~ /.well-known/acme-challenge/ {
         default_type "text/plain";
         root /openlibrary/static;