NUTCH-3187 - protocol-okhttp: IP address filter fails to compare first byte for /32 resp. /128 CIDRs

src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/CIDR.java

@@ -27,10 +27,16 @@
  * defined by the CIDR.
  */
 public class CIDR {
-  InetAddress addr;
-  int mask;
 
-  public CIDR(InetAddress address, int mask) {
+  private final InetAddress addr;
+  private final int mask;
+
+  public CIDR(InetAddress address, int mask) throws IllegalArgumentException {
+    int maxMask = address.getAddress().length * 8;
+    if (mask < 0 || mask > maxMask) {
+      throw new IllegalArgumentException(
+          "Invalid CIDR mask /" + mask + " for " + address);
+    }
     this.addr = address;
     this.mask = mask;
   }
@@ -42,16 +48,23 @@ public CIDR(String cidr) throws IllegalArgumentException {
       ipStr = cidr.substring(0, sep);
     }
     addr = InetAddresses.forString(ipStr);
+    int parsedMask;
     if (sep > -1) {
-      mask = Integer.parseInt(cidr.substring(sep + 1));
+      parsedMask = Integer.parseInt(cidr.substring(sep + 1));
     } else {
-      mask = addr.getAddress().length * 8;
+      parsedMask = addr.getAddress().length * 8;
     }
     if (cidr.indexOf(':') > -1 && addr.getAddress().length == 4) {
       // IPv4-mapped IPv6 addresses are automatically converted to IPv4,
       // need to shift the mask
-      mask = Math.max(0, mask - 96);
+      parsedMask = Math.max(0, parsedMask - 96);
+    }
+    int maxMask = addr.getAddress().length * 8;
+    if (parsedMask < 0 || parsedMask > maxMask) {
+      throw new IllegalArgumentException(
+          "Invalid CIDR mask /" + parsedMask + " for " + ipStr);
     }
+    this.mask = parsedMask;
   }
 
   public boolean contains(InetAddress address) {
@@ -63,11 +76,18 @@ public boolean contains(InetAddress address) {
     }
     for (int i = 0; i < addr0.length; i++) {
       int remainingMaskBits = mask - (i * 8);
-      if (remainingMaskBits <= 0)
+      if (remainingMaskBits <= 0) {
         return true;
-      int m = ~(0xff >> remainingMaskBits); // mask for byte under cursor
-      if ((addr0[i] & m) != (addr1[i] & m))
+      }
+      /*
+       * keep the mask within one byte so the shift does not wrap (Java shifts
+       * mod 32)
+       */
+      int m = remainingMaskBits >= 8 ? 0xff
+          : (0xff << (8 - remainingMaskBits)) & 0xff;
+      if ((addr0[i] & m) != (addr1[i] & m)) {
         return false;
+      }
     }
     return true;
   }

src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/IPFilterRules.java

@@ -30,7 +30,7 @@
  * Optionally limit or block connections to IP address ranges
  * (localhost/loopback or site-local addresses, subnet ranges given in CIDR
  * notation, or single IP addresses).
- * 
+ *
  * IP filter rules are built from two Nutch properties:
  * <ul>
  * <li><code>http.filter.ipaddress.include</code> defines all allowed IP ranges.
@@ -104,15 +104,15 @@ private static List<Predicate<InetAddress>> parseIPRules(Configuration conf,
       switch (ipRule.toLowerCase()) {
       case "localhost":
       case "loopback":
-        rules.add((InetAddress a) -> a.isLoopbackAddress());
+        rules.add(InetAddress::isLoopbackAddress);
         break;
       case "sitelocal":
-        rules.add((InetAddress a) -> a.isSiteLocalAddress());
+        rules.add(InetAddress::isSiteLocalAddress);
         break;
       default:
         try {
           CIDR cidr = new CIDR(ipRule);
-          rules.add((InetAddress a) -> cidr.contains(a));
+          rules.add(cidr::contains);
         } catch (IllegalArgumentException e) {
           LOG.error(
               "Failed to parse {} as CIDR, ignoring to configure IP rules ({})",

src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/TestIPAddressFiltering.java

@@ -16,9 +16,6 @@
  */
 package org.apache.nutch.protocol.okhttp;
 
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.junit.jupiter.api.Assertions.*;
-
 import java.net.InetAddress;
 import java.util.function.Function;
 
@@ -28,6 +25,12 @@
 
 import com.google.common.net.InetAddresses;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
 /**
  * Test cases for protocol-okhttp IP address filtering
  */
@@ -60,6 +63,17 @@ public void testCIDRnotContains(String cidr, String ip) {
     assertFalse(c.contains(i), i + " should not be in " + c);
   }
 
+  @Test
+  public void testInvalidCIDR() {
+    assertThrows(IllegalArgumentException.class,
+        () -> new CIDR(InetAddress.getByName("1.2.3.4"), 33));
+    assertThrows(IllegalArgumentException.class, () -> new CIDR("1.2.3.4/33"));
+    assertThrows(IllegalArgumentException.class, () -> new CIDR("::ffff:7f00:0/129"));
+    assertThrows(IllegalArgumentException.class, () -> new CIDR("1.2.3.4/-1"));
+    // invalid rule
+    assertThrows(IllegalArgumentException.class, () -> new CIDR("foobar"));
+  }
+
   /** Tests for {@link CIDR} */
   @Test
   public void testCIDRs() {
@@ -86,6 +100,13 @@ public void testCIDRs() {
     testCIDRnotContains("10.0.0.0/8", "9.255.255.255");
     testCIDRnotContains("172.16.0.0/12", "172.32.0.0");
     testCIDRnotContains("172.16.0.0/12", "171.255.255.255");
+
+    testCIDRnotContains("127.0.0.1/31", "0.0.0.1");
+    // NUTCH-3187 CIDR edge cases - /32, /128 - test bit shifting
+    testCIDRnotContains("127.0.0.1", "0.0.0.1");
+    testCIDRnotContains("127.0.0.1/32", "0.0.0.1");
+    testCIDRnotContains("127.0.0.1", "255.0.0.1");
+    testCIDRnotContains("fe80::2f29:b6f0:a4c:32ae/128", "::2f29:b6f0:a4c:32ae");
   }
 
   public void testFilter(Configuration conf, String[] included, String[] excluded) {
@@ -123,7 +144,7 @@ public void testIPAddressFilterRules() {
     conf.set("http.filter.ipaddress.exclude", "");
     testFilter(conf, loopbackAddresses, publicAddresses);
   }
-  
+
   public void testPredefinedAddressRange(String ipAddress, String type) {
     try {
       InetAddress addr = InetAddresses.forString(ipAddress);