Merge pull request #11420 from cdrini/experiment/counter-link

Experiment with counter link

Author: cdrini

docker/nginx.conf

@@ -32,6 +32,9 @@ http {
     include /olsystem/etc/nginx/logging.conf;
     access_log    /var/log/nginx/access.log iacombined;
 
+    js_shared_dict_zone zone=crawler_ips:10M type=number timeout=300s;
+    js_import /olsystem/etc/nginx/tagger.js;
+
     client_max_body_size 50m;
 
     sendfile        on;
@@ -52,20 +55,42 @@ http {
     # These rules only do anything if invoked, e.g., in web_nginx.conf.
     # TLDR: these rules can be disabled in `docker/web_nginx.conf`
     # and `docker/covers_nginx.conf`.
-    geo $should_apply_limit {
-      # No rate limit when IP obfuscation is not applied, as every IP is 255.0.0.0.
-      255.0.0.0 0;
-      # In cluster traffic
-      207.241.224.0/20 0;
-      # All other traffic
-      default 1;
+    geo $is_blessed_ip {
+      255.0.0.0 1;         # Internal
+      207.241.224.0/20 1;  # In cluster traffic
+      default 0;           # All other traffic
     }
 
-    map $should_apply_limit $rate_limit_key {
-      0 '';
-      1 $binary_remote_addr;
+    # Provides $is_blessed_ua
+    include /olsystem/etc/nginx/is_blessed_ua.map;
+
+    map "$is_blessed_ip:$is_blessed_ua" $rate_limit_key {
+      "0:0" $binary_remote_addr;  # Rate-limit by IP
+      default '';  # Don't rate-limit
     }
 
+    # check if user-agent provides a means of identification
+    map $http_user_agent $is_identifying_ua {
+        default 0;
+        "~*bot" 1;
+        "~*spider" 1;
+        "~*crawl" 1;
+        "~*google" 1; # sometimes just GoogleOther
+        "~*http" 1;   # Includes url
+        "~*@" 1;      # Includes email
+     }
+
+    js_set $has_hit_crawler_links tagger.check;
+
+    # The only crawlers we want to limit are the ones that don't identify themselves as such
+    map "$is_blessed_ip:$is_identifying_ua:$has_hit_crawler_links" $global_nonidentifying_crawler_rate_limit_key {
+        default '';  # No shared rate limiting
+        "0:0:1" '1'; # Shared rate limit
+    }
+
+    # Limit the crawlers that scrape links but don't ID themselves globally
+    limit_req_zone $global_nonidentifying_crawler_rate_limit_key zone=global_crawler_limit:5m rate=15r/s;
+
     # Matches other sites
     limit_req_zone $rate_limit_key zone=web_limit:10m rate=1r/s;
     # Higher rate for APIs since they are cheaper and we often hit them

docker/web_nginx.conf

@@ -105,8 +105,12 @@ server {
 
     location / {
         limit_req zone=web_limit burst=100 delay=10;
+        limit_req zone=global_crawler_limit nodelay;
         limit_req_status 429;
 
+        js_set $is_crawler_link tagger.tag_crawler;
+        add_header X-SPS $is_crawler_link;  # Need to reference the variable for the js method to be executed
+
         # For returning 200 when someone tries to randomly sort author results.
         if ($is_sus_random_sort) {
             return 200;
@@ -117,7 +121,7 @@ server {
         }
 
         if ($is_sus_referer) {
-            return 444;
+            return 403;
         }
 
         # Haproxy to better handle load/traffic
@@ -138,7 +142,7 @@ server {
         limit_req_status 429;
 
         if ($http_user_agent ~* (bytespider|meta-externalagent) ) {
-            return 444;
+            return 403;
         }
 
         # Haproxy to better handle load/traffic

openlibrary/templates/lib/nav_foot.html

@@ -44,6 +44,8 @@ <h2>$_("Help")</h2>
           <li><a href="/help/faq/editing" title="$_('Suggest Edits')">$_("Suggesting Edits")</a></li>
           <li><a href="/books/add" title="$_('Add a new book to Open Library')">$_("Add a Book")</a></li>
           <li><a href="https://github.com/internetarchive/openlibrary/releases" title="$_('Release Notes')">$_("Release Notes")</a></li>
+          $# detect-missing-i18n-skip-line
+          <li><a href="$changequery(dict(show_page_status=1))" style="color:transparent;position:absolute;pointer-events:none;" tabindex="-1">Page Status</a></li>
         </ul>
         <aside>
           <a class="footer-icon" title="$_('Twitter')" href="https://twitter.com/OpenLibrary"><img src="/static/images/tweet.svg" alt="" loading="lazy"></a>