Merge pull request #11605 from cdrini/fix/counter-link-improvements

Tweak tagger/nginx rate limit logic

Author: jimchamp

docker/covers_nginx.conf

@@ -66,6 +66,7 @@ server {
 
         # Covers rate limit.
         limit_req zone=cover_limit burst=400 nodelay;
+        limit_req zone=global_crawler_cover_limit nodelay;
         limit_req_status 429;
       }
 

docker/nginx.conf

@@ -61,8 +61,8 @@ http {
       default 0;           # All other traffic
     }
 
-    # Provides $is_blessed_ua
-    include /olsystem/etc/nginx/is_blessed_ua.map;
+    include /olsystem/etc/nginx/is_blessed_ua.conf;  # Provides $is_blessed_ua
+    include /olsystem/etc/nginx/is_sus_ip.conf;  # Provides $is_sus_ip
 
     map "$is_blessed_ip:$is_blessed_ua" $rate_limit_key {
       "0:0" $binary_remote_addr;  # Rate-limit by IP
@@ -83,13 +83,16 @@ http {
     js_set $has_hit_crawler_links tagger.check;
 
     # The only crawlers we want to limit are the ones that don't identify themselves as such
-    map "$is_blessed_ip:$is_identifying_ua:$has_hit_crawler_links" $global_nonidentifying_crawler_rate_limit_key {
+    map "$is_blessed_ip:$is_identifying_ua:$has_hit_crawler_links:$is_sus_ip" $global_nonidentifying_crawler_rate_limit_key {
         default '';  # No shared rate limiting
-        "0:0:1" '1'; # Shared rate limit
+        # Shared rate limit
+        "0:0:1:0" 1;
+        "0:0:1:1" 1;
+        "0:0:0:1" 1;
     }
 
     # Limit the crawlers that scrape links but don't ID themselves globally
-    limit_req_zone $global_nonidentifying_crawler_rate_limit_key zone=global_crawler_limit:5m rate=15r/s;
+    limit_req_zone $global_nonidentifying_crawler_rate_limit_key zone=global_crawler_limit:5m rate=17r/s;
 
     # Matches other sites
     limit_req_zone $rate_limit_key zone=web_limit:10m rate=1r/s;
@@ -98,6 +101,7 @@ http {
     limit_req_zone $rate_limit_key zone=api_limit:10m rate=180r/m;
     # Set a more permissive limit for covers because some pages might load 20+ covers.
     limit_req_zone $rate_limit_key zone=cover_limit:10m rate=400r/m;
+    limit_req_zone $global_nonidentifying_crawler_rate_limit_key zone=global_crawler_cover_limit:5m rate=150r/s;
 
     # Things are mounted into here by the docker compose file
     include /etc/nginx/sites-enabled/*;

docker/web_nginx.conf

@@ -16,6 +16,7 @@ map $request_uri$args $is_sus_random_sort {
 # 1. Detect empty referer
 map $http_referer $is_empty_referer {
     default 0;
+    "-" 1;
     "" 1;
 }
 
@@ -26,13 +27,16 @@ map $cookie_session $is_logged_out {
 }
 
 # 3. Check if path requires referer or login
-map $request_uri$args $requires_referer {
+map $request_uri $requires_referer {
     default 0;
     "~*^/(qrcode|admin|wp-login|show-records)" 1;
     "~*/edit(/|$|\?)" 1;
     "~*(v|m|action|redirect)=" 1;
+    "~*^/(subjects|authors)/" 1;
+    "~*^/search\?" 1;
 }
 
+# This is used by olsystem etc/nginx/tagger.js
 map "$is_empty_referer$is_logged_out$requires_referer" $is_sus_referer {
     default 0;
     "111" 1;
@@ -120,10 +124,6 @@ server {
             return 403;
         }
 
-        if ($is_sus_referer) {
-            return 403;
-        }
-
         # Haproxy to better handle load/traffic
         proxy_pass http://web_haproxy:7072;
         proxy_set_header Host $http_host;