diff --git a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/CIDR.java b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/CIDR.java
index 3add082a81..1d97e67280 100644
--- a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/CIDR.java
+++ b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/CIDR.java
@@ -27,10 +27,16 @@
* defined by the CIDR.
*/
public class CIDR {
- InetAddress addr;
- int mask;
- public CIDR(InetAddress address, int mask) {
+ private final InetAddress addr;
+ private final int mask;
+
+ public CIDR(InetAddress address, int mask) throws IllegalArgumentException {
+ int maxMask = address.getAddress().length * 8;
+ if (mask < 0 || mask > maxMask) {
+ throw new IllegalArgumentException(
+ "Invalid CIDR mask /" + mask + " for " + address);
+ }
this.addr = address;
this.mask = mask;
}
@@ -42,16 +48,23 @@ public CIDR(String cidr) throws IllegalArgumentException {
ipStr = cidr.substring(0, sep);
}
addr = InetAddresses.forString(ipStr);
+ int parsedMask;
if (sep > -1) {
- mask = Integer.parseInt(cidr.substring(sep + 1));
+ parsedMask = Integer.parseInt(cidr.substring(sep + 1));
} else {
- mask = addr.getAddress().length * 8;
+ parsedMask = addr.getAddress().length * 8;
}
if (cidr.indexOf(':') > -1 && addr.getAddress().length == 4) {
// IPv4-mapped IPv6 addresses are automatically converted to IPv4,
// need to shift the mask
- mask = Math.max(0, mask - 96);
+ parsedMask = Math.max(0, parsedMask - 96);
+ }
+ int maxMask = addr.getAddress().length * 8;
+ if (parsedMask < 0 || parsedMask > maxMask) {
+ throw new IllegalArgumentException(
+ "Invalid CIDR mask /" + parsedMask + " for " + ipStr);
}
+ this.mask = parsedMask;
}
public boolean contains(InetAddress address) {
@@ -63,11 +76,18 @@ public boolean contains(InetAddress address) {
}
for (int i = 0; i < addr0.length; i++) {
int remainingMaskBits = mask - (i * 8);
- if (remainingMaskBits <= 0)
+ if (remainingMaskBits <= 0) {
return true;
- int m = ~(0xff >> remainingMaskBits); // mask for byte under cursor
- if ((addr0[i] & m) != (addr1[i] & m))
+ }
+ /*
+ * keep the mask within one byte so the shift does not wrap (Java shifts
+ * mod 32)
+ */
+ int m = remainingMaskBits >= 8 ? 0xff
+ : (0xff << (8 - remainingMaskBits)) & 0xff;
+ if ((addr0[i] & m) != (addr1[i] & m)) {
return false;
+ }
}
return true;
}
diff --git a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/IPFilterRules.java b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/IPFilterRules.java
index 868732fe5e..9352ad74c3 100644
--- a/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/IPFilterRules.java
+++ b/src/plugin/protocol-okhttp/src/java/org/apache/nutch/protocol/okhttp/IPFilterRules.java
@@ -30,7 +30,7 @@
* Optionally limit or block connections to IP address ranges
* (localhost/loopback or site-local addresses, subnet ranges given in CIDR
* notation, or single IP addresses).
- *
+ *
* IP filter rules are built from two Nutch properties:
*
* http.filter.ipaddress.include defines all allowed IP ranges.
@@ -104,15 +104,15 @@ private static List> parseIPRules(Configuration conf,
switch (ipRule.toLowerCase()) {
case "localhost":
case "loopback":
- rules.add((InetAddress a) -> a.isLoopbackAddress());
+ rules.add(InetAddress::isLoopbackAddress);
break;
case "sitelocal":
- rules.add((InetAddress a) -> a.isSiteLocalAddress());
+ rules.add(InetAddress::isSiteLocalAddress);
break;
default:
try {
CIDR cidr = new CIDR(ipRule);
- rules.add((InetAddress a) -> cidr.contains(a));
+ rules.add(cidr::contains);
} catch (IllegalArgumentException e) {
LOG.error(
"Failed to parse {} as CIDR, ignoring to configure IP rules ({})",
diff --git a/src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/TestIPAddressFiltering.java b/src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/TestIPAddressFiltering.java
index 850b088186..62ecb332c9 100644
--- a/src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/TestIPAddressFiltering.java
+++ b/src/plugin/protocol-okhttp/src/test/org/apache/nutch/protocol/okhttp/TestIPAddressFiltering.java
@@ -16,9 +16,6 @@
*/
package org.apache.nutch.protocol.okhttp;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.junit.jupiter.api.Assertions.*;
-
import java.net.InetAddress;
import java.util.function.Function;
@@ -28,6 +25,12 @@
import com.google.common.net.InetAddresses;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
/**
* Test cases for protocol-okhttp IP address filtering
*/
@@ -60,6 +63,17 @@ public void testCIDRnotContains(String cidr, String ip) {
assertFalse(c.contains(i), i + " should not be in " + c);
}
+ @Test
+ public void testInvalidCIDR() {
+ assertThrows(IllegalArgumentException.class,
+ () -> new CIDR(InetAddress.getByName("1.2.3.4"), 33));
+ assertThrows(IllegalArgumentException.class, () -> new CIDR("1.2.3.4/33"));
+ assertThrows(IllegalArgumentException.class, () -> new CIDR("::ffff:7f00:0/129"));
+ assertThrows(IllegalArgumentException.class, () -> new CIDR("1.2.3.4/-1"));
+ // invalid rule
+ assertThrows(IllegalArgumentException.class, () -> new CIDR("foobar"));
+ }
+
/** Tests for {@link CIDR} */
@Test
public void testCIDRs() {
@@ -86,6 +100,13 @@ public void testCIDRs() {
testCIDRnotContains("10.0.0.0/8", "9.255.255.255");
testCIDRnotContains("172.16.0.0/12", "172.32.0.0");
testCIDRnotContains("172.16.0.0/12", "171.255.255.255");
+
+ testCIDRnotContains("127.0.0.1/31", "0.0.0.1");
+ // NUTCH-3187 CIDR edge cases - /32, /128 - test bit shifting
+ testCIDRnotContains("127.0.0.1", "0.0.0.1");
+ testCIDRnotContains("127.0.0.1/32", "0.0.0.1");
+ testCIDRnotContains("127.0.0.1", "255.0.0.1");
+ testCIDRnotContains("fe80::2f29:b6f0:a4c:32ae/128", "::2f29:b6f0:a4c:32ae");
}
public void testFilter(Configuration conf, String[] included, String[] excluded) {
@@ -123,7 +144,7 @@ public void testIPAddressFilterRules() {
conf.set("http.filter.ipaddress.exclude", "");
testFilter(conf, loopbackAddresses, publicAddresses);
}
-
+
public void testPredefinedAddressRange(String ipAddress, String type) {
try {
InetAddress addr = InetAddresses.forString(ipAddress);