Skip to content

Commit b7fe4e7

Browse files
ip2locationip2location
committed
Reduce I/O
1 parent 508a077 commit b7fe4e7

File tree

1 file changed

+114
-53
lines changed

1 file changed

+114
-53
lines changed

‎com/ip2proxy/IP2Proxy.java

+114-53
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import java.util.*;
99
import java.util.regex.*;
1010
import java.math.BigInteger;
11+
import java.nio.ByteBuffer;
1112
import java.nio.MappedByteBuffer;
1213
import java.nio.ByteOrder;
1314
import java.nio.channels.FileChannel;
@@ -114,7 +115,7 @@ private enum Modes {
114115
private boolean AS_ENABLED;
115116
private boolean LASTSEEN_ENABLED;
116117

117-
private static final String _ModuleVersion = "2.1.0";
118+
private static final String _ModuleVersion = "2.2.0";
118119

119120
public IP2Proxy() {
120121

@@ -372,16 +373,29 @@ private boolean LoadBIN() throws IOException {
372373
_IPv6ColumnSize = 16 + ((_DBColumn - 1) << 2); // 4 bytes each column, except IPFrom column which is 16 bytes
373374

374375
// since both IPv4 and IPv6 use 4 bytes for the below columns, can just do it once here
375-
COUNTRY_POSITION_OFFSET = (COUNTRY_POSITION[_DBType] != 0) ? (COUNTRY_POSITION[_DBType] - 1) << 2 : 0;
376-
REGION_POSITION_OFFSET = (REGION_POSITION[_DBType] != 0) ? (REGION_POSITION[_DBType] - 1) << 2 : 0;
377-
CITY_POSITION_OFFSET = (CITY_POSITION[_DBType] != 0) ? (CITY_POSITION[_DBType] - 1) << 2 : 0;
378-
ISP_POSITION_OFFSET = (ISP_POSITION[_DBType] != 0) ? (ISP_POSITION[_DBType] - 1) << 2 : 0;
379-
PROXYTYPE_POSITION_OFFSET = (PROXYTYPE_POSITION[_DBType] != 0) ? (PROXYTYPE_POSITION[_DBType] - 1) << 2 : 0;
380-
DOMAIN_POSITION_OFFSET = (DOMAIN_POSITION[_DBType] != 0) ? (DOMAIN_POSITION[_DBType] - 1) << 2 : 0;
381-
USAGETYPE_POSITION_OFFSET = (USAGETYPE_POSITION[_DBType] != 0) ? (USAGETYPE_POSITION[_DBType] - 1) << 2 : 0;
382-
ASN_POSITION_OFFSET = (ASN_POSITION[_DBType] != 0) ? (ASN_POSITION[_DBType] - 1) << 2 : 0;
383-
AS_POSITION_OFFSET = (AS_POSITION[_DBType] != 0) ? (AS_POSITION[_DBType] - 1) << 2 : 0;
384-
LASTSEEN_POSITION_OFFSET = (LASTSEEN_POSITION[_DBType] != 0) ? (LASTSEEN_POSITION[_DBType] - 1) << 2 : 0;
376+
// COUNTRY_POSITION_OFFSET = (COUNTRY_POSITION[_DBType] != 0) ? (COUNTRY_POSITION[_DBType] - 1) << 2 : 0;
377+
// REGION_POSITION_OFFSET = (REGION_POSITION[_DBType] != 0) ? (REGION_POSITION[_DBType] - 1) << 2 : 0;
378+
// CITY_POSITION_OFFSET = (CITY_POSITION[_DBType] != 0) ? (CITY_POSITION[_DBType] - 1) << 2 : 0;
379+
// ISP_POSITION_OFFSET = (ISP_POSITION[_DBType] != 0) ? (ISP_POSITION[_DBType] - 1) << 2 : 0;
380+
// PROXYTYPE_POSITION_OFFSET = (PROXYTYPE_POSITION[_DBType] != 0) ? (PROXYTYPE_POSITION[_DBType] - 1) << 2 : 0;
381+
// DOMAIN_POSITION_OFFSET = (DOMAIN_POSITION[_DBType] != 0) ? (DOMAIN_POSITION[_DBType] - 1) << 2 : 0;
382+
// USAGETYPE_POSITION_OFFSET = (USAGETYPE_POSITION[_DBType] != 0) ? (USAGETYPE_POSITION[_DBType] - 1) << 2 : 0;
383+
// ASN_POSITION_OFFSET = (ASN_POSITION[_DBType] != 0) ? (ASN_POSITION[_DBType] - 1) << 2 : 0;
384+
// AS_POSITION_OFFSET = (AS_POSITION[_DBType] != 0) ? (AS_POSITION[_DBType] - 1) << 2 : 0;
385+
// LASTSEEN_POSITION_OFFSET = (LASTSEEN_POSITION[_DBType] != 0) ? (LASTSEEN_POSITION[_DBType] - 1) << 2 : 0;
386+
387+
// slightly different offset for reading by row
388+
COUNTRY_POSITION_OFFSET = (COUNTRY_POSITION[_DBType] != 0) ? (COUNTRY_POSITION[_DBType] - 2) << 2 : 0;
389+
REGION_POSITION_OFFSET = (REGION_POSITION[_DBType] != 0) ? (REGION_POSITION[_DBType] - 2) << 2 : 0;
390+
CITY_POSITION_OFFSET = (CITY_POSITION[_DBType] != 0) ? (CITY_POSITION[_DBType] - 2) << 2 : 0;
391+
ISP_POSITION_OFFSET = (ISP_POSITION[_DBType] != 0) ? (ISP_POSITION[_DBType] - 2) << 2 : 0;
392+
PROXYTYPE_POSITION_OFFSET = (PROXYTYPE_POSITION[_DBType] != 0) ? (PROXYTYPE_POSITION[_DBType] - 2) << 2 : 0;
393+
DOMAIN_POSITION_OFFSET = (DOMAIN_POSITION[_DBType] != 0) ? (DOMAIN_POSITION[_DBType] - 2) << 2 : 0;
394+
USAGETYPE_POSITION_OFFSET = (USAGETYPE_POSITION[_DBType] != 0) ? (USAGETYPE_POSITION[_DBType] - 2) << 2 : 0;
395+
ASN_POSITION_OFFSET = (ASN_POSITION[_DBType] != 0) ? (ASN_POSITION[_DBType] - 2) << 2 : 0;
396+
AS_POSITION_OFFSET = (AS_POSITION[_DBType] != 0) ? (AS_POSITION[_DBType] - 2) << 2 : 0;
397+
LASTSEEN_POSITION_OFFSET = (LASTSEEN_POSITION[_DBType] != 0) ? (LASTSEEN_POSITION[_DBType] - 2) << 2 : 0;
398+
385399

386400
COUNTRY_ENABLED = (COUNTRY_POSITION[_DBType] != 0) ? true : false;
387401
REGION_ENABLED = (REGION_POSITION[_DBType] != 0) ? true : false;
@@ -477,7 +491,9 @@ public ProxyResult ProxyQuery(String IPAddress) throws IOException {
477491
public ProxyResult ProxyQuery(String IPAddress, Modes Mode) throws IOException {
478492
ProxyResult Result = new ProxyResult();
479493
RandomAccessFile RF = null;
480-
MappedByteBuffer Buf = null;
494+
// MappedByteBuffer Buf = null;
495+
ByteBuffer Buf = null;
496+
ByteBuffer DataBuf = null;
481497

482498
try {
483499
if (IPAddress == null || IPAddress.length() == 0) {
@@ -538,7 +554,7 @@ public ProxyResult ProxyQuery(String IPAddress, Modes Mode) throws IOException {
538554
return Result;
539555
}
540556

541-
long CountryPos = 0;
557+
long Pos = 0;
542558
long Low = 0;
543559
long High = 0;
544560
long Mid = 0;
@@ -579,7 +595,9 @@ public ProxyResult ProxyQuery(String IPAddress, Modes Mode) throws IOException {
579595
High = _DBCount;
580596

581597
if (_UseMemoryMappedFile) {
582-
Buf = _IPv4Buffer;
598+
// Buf = _IPv4Buffer;
599+
Buf = _IPv4Buffer.duplicate(); // this enables this thread to maintain its own position in a multi-threaded environment
600+
Buf.order(ByteOrder.LITTLE_ENDIAN);
583601
BufCapacity = Buf.capacity();
584602
}
585603
else {
@@ -611,7 +629,9 @@ public ProxyResult ProxyQuery(String IPAddress, Modes Mode) throws IOException {
611629
High = _DBCountIPv6;
612630

613631
if (_UseMemoryMappedFile) {
614-
Buf = _IPv6Buffer;
632+
// Buf = _IPv6Buffer;
633+
Buf = _IPv6Buffer.duplicate(); // this enables this thread to maintain its own position in a multi-threaded environment
634+
Buf.order(ByteOrder.LITTLE_ENDIAN);
615635
BufCapacity = Buf.capacity();
616636
}
617637
else {
@@ -654,73 +674,97 @@ public ProxyResult ProxyQuery(String IPAddress, Modes Mode) throws IOException {
654674
String AS = MSG_NOT_SUPPORTED;
655675
String Last_Seen = MSG_NOT_SUPPORTED;
656676

677+
int FirstCol = 4; // IP From is 4 bytes
657678
if (IPType == 6) { // IPv6
658-
RowOffset = RowOffset + 12; // coz below is assuming all columns are 4 bytes, so got 12 left to go to make 16 bytes total
679+
FirstCol = 16; // IPv6 is 16 bytes
680+
// RowOffset = RowOffset + 12; // coz below is assuming all columns are 4 bytes, so got 12 left to go to make 16 bytes total
681+
}
682+
683+
// read the row here after the IP From column (remaining columns are all 4 bytes)
684+
int RowLen = ColumnSize - FirstCol;
685+
byte[] Row;
686+
Row = ReadRow(RowOffset + FirstCol, RowLen, Buf, RF);
687+
688+
if (_UseMemoryMappedFile) {
689+
DataBuf = _MapDataBuffer.duplicate(); // this is to enable reading of a range of bytes in multi-threaded environment
690+
DataBuf.order(ByteOrder.LITTLE_ENDIAN);
659691
}
660692

661693
if (PROXYTYPE_ENABLED) {
662694
if (Mode == Modes.ALL || Mode == Modes.PROXY_TYPE || Mode == Modes.IS_PROXY) {
663-
Proxy_Type = ReadStr(Read32(RowOffset + PROXYTYPE_POSITION_OFFSET, Buf, RF).longValue(), RF);
695+
// Proxy_Type = ReadStr(Read32(RowOffset + PROXYTYPE_POSITION_OFFSET, Buf, RF).longValue(), RF);
696+
Proxy_Type = ReadStr(Read32_Row(Row, PROXYTYPE_POSITION_OFFSET).longValue(), DataBuf, RF);
664697
}
665698
}
666699

667700
if (COUNTRY_ENABLED) {
668701
if (Mode == Modes.ALL || Mode == Modes.COUNTRY_SHORT || Mode == Modes.COUNTRY_LONG || Mode == Modes.IS_PROXY) {
669-
CountryPos = Read32(RowOffset + COUNTRY_POSITION_OFFSET, Buf, RF).longValue();
702+
// CountryPos = Read32(RowOffset + COUNTRY_POSITION_OFFSET, Buf, RF).longValue();
703+
Pos = Read32_Row(Row, COUNTRY_POSITION_OFFSET).longValue();
670704
}
671705
if (Mode == Modes.ALL || Mode == Modes.COUNTRY_SHORT || Mode == Modes.IS_PROXY) {
672-
Country_Short = ReadStr(CountryPos, RF);
706+
// Country_Short = ReadStr(CountryPos, RF);
707+
Country_Short = ReadStr(Pos, DataBuf, RF);
673708
}
674709
if (Mode == Modes.ALL || Mode == Modes.COUNTRY_LONG) {
675-
Country_Long = ReadStr(CountryPos + 3, RF);
710+
// Country_Long = ReadStr(CountryPos + 3, RF);
711+
Country_Long = ReadStr(Pos + 3, DataBuf, RF);
676712
}
677713
}
678714

679715
if (REGION_ENABLED) {
680716
if (Mode == Modes.ALL || Mode == Modes.REGION) {
681-
Region = ReadStr(Read32(RowOffset + REGION_POSITION_OFFSET, Buf, RF).longValue(), RF);
717+
// Region = ReadStr(Read32(RowOffset + REGION_POSITION_OFFSET, Buf, RF).longValue(), RF);
718+
Region = ReadStr(Read32_Row(Row, REGION_POSITION_OFFSET).longValue(), DataBuf, RF);
682719
}
683720
}
684721

685722
if (CITY_ENABLED) {
686723
if (Mode == Modes.ALL || Mode == Modes.CITY) {
687-
City = ReadStr(Read32(RowOffset + CITY_POSITION_OFFSET, Buf, RF).longValue(), RF);
724+
// City = ReadStr(Read32(RowOffset + CITY_POSITION_OFFSET, Buf, RF).longValue(), RF);
725+
City = ReadStr(Read32_Row(Row, CITY_POSITION_OFFSET).longValue(), DataBuf, RF);
688726
}
689727
}
690728

691729
if (ISP_ENABLED) {
692730
if (Mode == Modes.ALL || Mode == Modes.ISP) {
693-
ISP = ReadStr(Read32(RowOffset + ISP_POSITION_OFFSET, Buf, RF).longValue(), RF);
731+
// ISP = ReadStr(Read32(RowOffset + ISP_POSITION_OFFSET, Buf, RF).longValue(), RF);
732+
ISP = ReadStr(Read32_Row(Row, ISP_POSITION_OFFSET).longValue(), DataBuf, RF);
694733
}
695734
}
696735

697736
if (DOMAIN_ENABLED) {
698737
if (Mode == Modes.ALL || Mode == Modes.DOMAIN) {
699-
Domain = ReadStr(Read32(RowOffset + DOMAIN_POSITION_OFFSET, Buf, RF).longValue(), RF);
738+
// Domain = ReadStr(Read32(RowOffset + DOMAIN_POSITION_OFFSET, Buf, RF).longValue(), RF);
739+
Domain = ReadStr(Read32_Row(Row, DOMAIN_POSITION_OFFSET).longValue(), DataBuf, RF);
700740
}
701741
}
702742

703743
if (USAGETYPE_ENABLED) {
704744
if (Mode == Modes.ALL || Mode == Modes.USAGE_TYPE) {
705-
Usage_Type = ReadStr(Read32(RowOffset + USAGETYPE_POSITION_OFFSET, Buf, RF).longValue(), RF);
745+
// Usage_Type = ReadStr(Read32(RowOffset + USAGETYPE_POSITION_OFFSET, Buf, RF).longValue(), RF);
746+
Usage_Type = ReadStr(Read32_Row(Row, USAGETYPE_POSITION_OFFSET).longValue(), DataBuf, RF);
706747
}
707748
}
708749

709750
if (ASN_ENABLED) {
710751
if (Mode == Modes.ALL || Mode == Modes.ASN) {
711-
ASN = ReadStr(Read32(RowOffset + ASN_POSITION_OFFSET, Buf, RF).longValue(), RF);
752+
// ASN = ReadStr(Read32(RowOffset + ASN_POSITION_OFFSET, Buf, RF).longValue(), RF);
753+
ASN = ReadStr(Read32_Row(Row, ASN_POSITION_OFFSET).longValue(), DataBuf, RF);
712754
}
713755
}
714756

715757
if (AS_ENABLED) {
716758
if (Mode == Modes.ALL || Mode == Modes.AS) {
717-
AS = ReadStr(Read32(RowOffset + AS_POSITION_OFFSET, Buf, RF).longValue(), RF);
759+
// AS = ReadStr(Read32(RowOffset + AS_POSITION_OFFSET, Buf, RF).longValue(), RF);
760+
AS = ReadStr(Read32_Row(Row, AS_POSITION_OFFSET).longValue(), DataBuf, RF);
718761
}
719762
}
720763

721764
if (LASTSEEN_ENABLED) {
722765
if (Mode == Modes.ALL || Mode == Modes.LAST_SEEN) {
723-
Last_Seen = ReadStr(Read32(RowOffset + LASTSEEN_POSITION_OFFSET, Buf, RF).longValue(), RF);
766+
// Last_Seen = ReadStr(Read32(RowOffset + LASTSEEN_POSITION_OFFSET, Buf, RF).longValue(), RF);
767+
Last_Seen = ReadStr(Read32_Row(Row, LASTSEEN_POSITION_OFFSET).longValue(), DataBuf, RF);
724768
}
725769
}
726770

@@ -1011,7 +1055,20 @@ private void Reverse(byte[] Arr) {
10111055
}
10121056
}
10131057

1014-
private BigInteger Read32Or128(final long Position, final int IPType, final MappedByteBuffer Buf, final RandomAccessFile RH) throws IOException {
1058+
private byte[] ReadRow(final long Position, final long MyLen, final ByteBuffer Buf, final RandomAccessFile RH) throws IOException {
1059+
byte[] Row = new byte[(int)MyLen];
1060+
if (_UseMemoryMappedFile) {
1061+
Buf.position((int)Position);
1062+
Buf.get(Row, (int)0, (int)MyLen);
1063+
}
1064+
else {
1065+
RH.seek(Position - 1);
1066+
RH.read(Row, (int)0, (int)MyLen);
1067+
}
1068+
return Row;
1069+
}
1070+
1071+
private BigInteger Read32Or128(final long Position, final int IPType, final ByteBuffer Buf, final RandomAccessFile RH) throws IOException {
10151072
if (IPType == 4) {
10161073
return Read32(Position, Buf, RH);
10171074
}
@@ -1021,28 +1078,33 @@ else if (IPType == 6) {
10211078
return BigInteger.ZERO;
10221079
}
10231080

1024-
private BigInteger Read128(final long Position, final MappedByteBuffer Buf, final RandomAccessFile RH) throws IOException {
1081+
private BigInteger Read128(final long Position, final ByteBuffer Buf, final RandomAccessFile RH) throws IOException {
10251082
BigInteger RetVal = BigInteger.ZERO;
10261083
final int BSize = 16;
10271084
byte Bytes[] = new byte[BSize];
10281085

10291086
if (_UseMemoryMappedFile) {
1030-
for (int x = 0; x < BSize; x++) {
1031-
Bytes[x] = Buf.get((int)Position + x); // use absolute offset to be thread-safe
1032-
}
1087+
Buf.position((int)Position);
1088+
Buf.get(Bytes, (int)0, BSize);
10331089
}
10341090
else {
10351091
RH.seek(Position - 1);
1036-
for (int x = 0; x < BSize; x++) {
1037-
Bytes[x] = RH.readByte();
1038-
}
1092+
RH.read(Bytes, (int)0, BSize);
10391093
}
10401094
Reverse(Bytes);
10411095
RetVal = new BigInteger(1, Bytes);
10421096
return RetVal;
10431097
}
1044-
1045-
private BigInteger Read32(final long Position, final MappedByteBuffer Buf, final RandomAccessFile RH) throws IOException {
1098+
1099+
private BigInteger Read32_Row(byte[] Row, final int From) throws IOException {
1100+
final int Len = 4; // 4 bytes
1101+
byte Bytes[] = new byte[Len];
1102+
System.arraycopy(Row, From, Bytes, (int)0, Len);
1103+
Reverse(Bytes);
1104+
return new BigInteger(1, Bytes);
1105+
}
1106+
1107+
private BigInteger Read32(final long Position, final ByteBuffer Buf, final RandomAccessFile RH) throws IOException {
10461108
if (_UseMemoryMappedFile) {
10471109
// simulate unsigned int by using long
10481110
return BigInteger.valueOf(Buf.getInt((int)Position) & 0xffffffffL); // use absolute offset to be thread-safe
@@ -1051,27 +1113,24 @@ private BigInteger Read32(final long Position, final MappedByteBuffer Buf, final
10511113
final int BSize = 4;
10521114
RH.seek(Position - 1);
10531115
byte Bytes[] = new byte[BSize];
1054-
for (int x = 0; x < BSize; x++) {
1055-
Bytes[x] = RH.readByte();
1056-
}
1116+
RH.read(Bytes, (int)0, BSize);
10571117
Reverse(Bytes);
10581118
return new BigInteger(1, Bytes);
10591119
}
10601120
}
1061-
1062-
private String ReadStr(long Position, final RandomAccessFile RH) throws IOException {
1121+
1122+
private String ReadStr(long Position, final ByteBuffer Buf, final RandomAccessFile RH) throws IOException {
10631123
final int Size;
1064-
char CBuf[] = null;
1124+
byte[] Bytes = null;
10651125

10661126
if (_UseMemoryMappedFile) {
10671127
Position = Position - _MapDataOffset; // position stored in BIN file is for full file, not just the mapped data segment, so need to minus
10681128
Size = _MapDataBuffer.get((int)Position); // use absolute offset to be thread-safe
10691129

10701130
try {
1071-
CBuf = new char[Size];
1072-
for (int x = 0; x < Size; x++) {
1073-
CBuf[x] = (char)_MapDataBuffer.get((int)Position + 1 + x); // use absolute offset to be thread-safe
1074-
}
1131+
Bytes = new byte[Size];
1132+
Buf.position((int)Position + 1);
1133+
Buf.get(Bytes, (int)0, Size);
10751134
}
10761135
catch (NegativeArraySizeException e) {
10771136
return null;
@@ -1081,16 +1140,16 @@ private String ReadStr(long Position, final RandomAccessFile RH) throws IOExcept
10811140
RH.seek(Position);
10821141
Size = RH.read();
10831142
try {
1084-
CBuf = new char[Size];
1085-
for (int x = 0; x < Size; x++) {
1086-
CBuf[x] = (char)RH.read();
1087-
}
1143+
Bytes = new byte[Size];
1144+
RH.read(Bytes, (int)0, Size);
10881145
}
10891146
catch (NegativeArraySizeException e) {
10901147
return null;
10911148
}
10921149
}
1093-
return String.copyValueOf(CBuf);
1150+
1151+
String S = new String(Bytes);
1152+
return S;
10941153
}
10951154

10961155
private BigInteger[] IP2No(String IP) throws UnknownHostException {
@@ -1125,12 +1184,14 @@ else if (IA instanceof Inet4Address) { // this will run in cases of IPv4-mapped
11251184
IPType = "4";
11261185
A2 = A2.shiftRight(80);
11271186
A2 = A2.and(LAST_32BITS);
1187+
A3 = new BigInteger("4");
11281188
}
11291189
else if (A2.compareTo(FROM_TEREDO) >= 0 && A2.compareTo(TO_TEREDO) <= 0) {
11301190
// Teredo so need to remap to ipv4
11311191
IPType = "4";
11321192
A2 = A2.not();
11331193
A2 = A2.and(LAST_32BITS);
1194+
A3 = new BigInteger("4");
11341195
}
11351196
A1 = new BigInteger(IPType);
11361197
}

0 commit comments

Comments
 (0)