Skip to content

Commit 123cec4

Browse files
ip2locationip2location
committed
Reduce file I/O
1 parent 0a719fb commit 123cec4

File tree

2 files changed

+73
-42
lines changed

2 files changed

+73
-42
lines changed

‎com/ip2proxy/Http.java

-1
Original file line numberDiff line numberDiff line change
@@ -31,5 +31,4 @@ public static String get(URL url) {
3131
throw new RuntimeException(e);
3232
}
3333
}
34-
3534
}

‎com/ip2proxy/IP2Proxy.java

+73-41
Original file line numberDiff line numberDiff line change
@@ -93,8 +93,8 @@ private enum Modes {
9393
private int _IndexBaseAddr = 0;
9494
private int _IndexBaseAddrIPv6 = 0;
9595
private int _ProductCode = 0;
96-
// private int _ProductType = 0;
97-
// private int _FileSize = 0;
96+
private int _ProductType = 0;
97+
private int _FileSize = 0;
9898

9999
private boolean _UseMemoryMappedFile = false;
100100
private String _IPDatabasePath = "";
@@ -125,7 +125,7 @@ private enum Modes {
125125
private boolean THREAT_ENABLED;
126126
private boolean PROVIDER_ENABLED;
127127

128-
private static final String _ModuleVersion = "3.2.0";
128+
private static final String _ModuleVersion = "3.3.0";
129129

130130
public IP2Proxy() {
131131

@@ -346,8 +346,8 @@ public int Close() {
346346
_IndexBaseAddr = 0;
347347
_IndexBaseAddrIPv6 = 0;
348348
_ProductCode = 0;
349-
// _ProductType = 0;
350-
// _FileSize = 0;
349+
_ProductType = 0;
350+
_FileSize = 0;
351351
return 0;
352352
}
353353

@@ -389,14 +389,14 @@ private void CreateMappedBytes(FileChannel InChannel) throws IOException {
389389

390390
private boolean LoadBIN() throws IOException {
391391
boolean LoadOK = false;
392-
RandomAccessFile RF = null;
392+
RandomAccessFile aFile = null;
393393

394394
try {
395395
if (_IPDatabasePath.length() > 0) {
396-
RF = new RandomAccessFile(_IPDatabasePath, "r");
397-
final FileChannel InChannel = RF.getChannel();
398-
final MappedByteBuffer _HeaderBuffer = InChannel.map(FileChannel.MapMode.READ_ONLY, 0, 64); // 64 bytes header
399-
396+
aFile = new RandomAccessFile(_IPDatabasePath, "r");
397+
byte[] _HeaderData = new byte[64];
398+
aFile.read(_HeaderData);
399+
ByteBuffer _HeaderBuffer = ByteBuffer.wrap(_HeaderData);
400400
_HeaderBuffer.order(ByteOrder.LITTLE_ENDIAN);
401401

402402
_DBType = _HeaderBuffer.get(0);
@@ -410,10 +410,10 @@ private boolean LoadBIN() throws IOException {
410410
_BaseAddrIPv6 = _HeaderBuffer.getInt(17); // 4 bytes
411411
_IndexBaseAddr = _HeaderBuffer.getInt(21); //4 bytes
412412
_IndexBaseAddrIPv6 = _HeaderBuffer.getInt(25); //4 bytes
413-
414413
_ProductCode = _HeaderBuffer.get(29);
415-
// _ProductType = _HeaderBuffer.get(31);
416-
// _FileSize = _HeaderBuffer.getInt(32); //4 bytes
414+
// below 2 fields just read for now, not being used yet
415+
_ProductType = _HeaderBuffer.get(30);
416+
_FileSize = _HeaderBuffer.getInt(31); //4 bytes
417417

418418
// check if is correct BIN (should be 2 for IP2Proxy BIN file), also checking for zipped file (PK being the first 2 chars)
419419
if ((_ProductCode != 2 && _DBYear >= 21) || (_DBType == 80 && _DBColumn == 75)) // only BINs from Jan 2021 onwards have this byte set
@@ -450,8 +450,17 @@ private boolean LoadBIN() throws IOException {
450450
THREAT_ENABLED = THREAT_POSITION[_DBType] != 0;
451451
PROVIDER_ENABLED = PROVIDER_POSITION[_DBType] != 0;
452452

453-
final MappedByteBuffer _IndexBuffer = InChannel.map(FileChannel.MapMode.READ_ONLY, _IndexBaseAddr - 1, _BaseAddr - _IndexBaseAddr);
453+
int readLen = _IndexArrayIPv4.length;
454+
if (_IndexBaseAddrIPv6 > 0) {
455+
readLen += _IndexArrayIPv6.length;
456+
}
457+
458+
byte[] _IndexData = new byte[readLen * 8]; // 4 bytes for both from row and to row
459+
aFile.seek(_IndexBaseAddr - 1);
460+
aFile.read(_IndexData);
461+
ByteBuffer _IndexBuffer = ByteBuffer.wrap(_IndexData);
454462
_IndexBuffer.order(ByteOrder.LITTLE_ENDIAN);
463+
455464
int Pointer = 0;
456465

457466
// read IPv4 index
@@ -471,15 +480,15 @@ private boolean LoadBIN() throws IOException {
471480
}
472481

473482
if (_UseMemoryMappedFile) {
474-
CreateMappedBytes(InChannel);
483+
CreateMappedBytes();
475484
} else {
476485
DestroyMappedBytes();
477486
}
478487
LoadOK = true;
479488
}
480489
} finally {
481-
if (RF != null) {
482-
RF.close();
490+
if (aFile != null) {
491+
aFile.close();
483492
}
484493
}
485494
return LoadOK;
@@ -538,6 +547,8 @@ public ProxyResult ProxyQuery(String IPAddress, Modes Mode) throws IOException {
538547
RandomAccessFile RF = null;
539548
ByteBuffer Buf = null;
540549
ByteBuffer DataBuf = null;
550+
byte[] Row;
551+
byte[] FullRow = null;
541552

542553
try {
543554
if (IPAddress == null || IPAddress.length() == 0) {
@@ -571,6 +582,7 @@ public ProxyResult ProxyQuery(String IPAddress, Modes Mode) throws IOException {
571582
BigInteger[] BI;
572583
boolean OverCapacity = false;
573584
String[] RetArr;
585+
int FirstCol = 4; // IP From is 4 bytes
574586

575587
try {
576588
BI = IP2No(IPAddress);
@@ -652,6 +664,7 @@ public ProxyResult ProxyQuery(String IPAddress, Modes Mode) throws IOException {
652664
Low = _IndexArrayIPv4[IndexAddr][0];
653665
High = _IndexArrayIPv4[IndexAddr][1];
654666
} else { // IPv6
667+
FirstCol = 16; // IPv6 is 16 bytes
655668
if (_DBCountIPv6 == 0) {
656669
Result.Is_Proxy = -1;
657670
Result.Proxy_Type = MSG_IPV6_UNSUPPORTED;
@@ -696,12 +709,17 @@ public ProxyResult ProxyQuery(String IPAddress, Modes Mode) throws IOException {
696709
RowOffset2 = RowOffset + ColumnSize;
697710

698711
if (_UseMemoryMappedFile) {
712+
// only reading the IP From fields
699713
OverCapacity = (RowOffset2 >= BufCapacity);
714+
IPFrom = Read32Or128(RowOffset, IPType, Buf, RF);
715+
IPTo = (OverCapacity) ? BigInteger.ZERO : Read32Or128(RowOffset2, IPType, Buf, RF);
716+
} else {
717+
// reading IP From + whole row + next IP From
718+
FullRow = ReadRow(RowOffset, ColumnSize + FirstCol, Buf, RF);
719+
IPFrom = Read32Or128Row(FullRow, 0, FirstCol);
720+
IPTo = (OverCapacity) ? BigInteger.ZERO : Read32Or128Row(FullRow, ColumnSize, FirstCol);
700721
}
701722

702-
IPFrom = Read32Or128(RowOffset, IPType, Buf, RF);
703-
IPTo = (OverCapacity) ? BigInteger.ZERO : Read32Or128(RowOffset2, IPType, Buf, RF);
704-
705723
if (IPNo.compareTo(IPFrom) >= 0 && IPNo.compareTo(IPTo) < 0) {
706724
int Is_Proxy = -1;
707725
String Proxy_Type = MSG_NOT_SUPPORTED;
@@ -718,19 +736,15 @@ public ProxyResult ProxyQuery(String IPAddress, Modes Mode) throws IOException {
718736
String Threat = MSG_NOT_SUPPORTED;
719737
String Provider = MSG_NOT_SUPPORTED;
720738

721-
int FirstCol = 4; // IP From is 4 bytes
722-
if (IPType == 6) { // IPv6
723-
FirstCol = 16; // IPv6 is 16 bytes
724-
}
725-
726-
// read the row here after the IP From column (remaining columns are all 4 bytes)
727739
int RowLen = ColumnSize - FirstCol;
728-
byte[] Row;
729-
Row = ReadRow(RowOffset + FirstCol, RowLen, Buf, RF);
730740

731741
if (_UseMemoryMappedFile) {
742+
Row = ReadRow(RowOffset + FirstCol, RowLen, Buf, RF);
732743
DataBuf = _MapDataBuffer.duplicate(); // this is to enable reading of a range of bytes in multi-threaded environment
733744
DataBuf.order(ByteOrder.LITTLE_ENDIAN);
745+
} else {
746+
Row = new byte[RowLen];
747+
System.arraycopy(FullRow, FirstCol, Row, (int) 0, RowLen); // extract the actual row data
734748
}
735749

736750
if (PROXYTYPE_ENABLED) {
@@ -1089,6 +1103,13 @@ private byte[] ReadRow(final long Position, final long MyLen, final ByteBuffer B
10891103
return Row;
10901104
}
10911105

1106+
private BigInteger Read32Or128Row(byte[] Row, final int From, final int Len) throws IOException {
1107+
byte[] Buf = new byte[Len];
1108+
System.arraycopy(Row, From, Buf, (int) 0, Len);
1109+
Reverse(Buf);
1110+
return new BigInteger(1, Buf);
1111+
}
1112+
10921113
private BigInteger Read32Or128(final long Position, final int IPType, final ByteBuffer Buf, final RandomAccessFile RH) throws IOException {
10931114
if (IPType == 4) {
10941115
return Read32(Position, Buf, RH);
@@ -1137,33 +1158,44 @@ private BigInteger Read32(final long Position, final ByteBuffer Buf, final Rando
11371158
}
11381159
}
11391160

1140-
private String ReadStr(long Position, final ByteBuffer Buf, final RandomAccessFile RH) throws IOException {
1141-
final int Size;
1142-
byte[] Bytes;
1161+
private String ReadStr(long Position, final ByteBuffer DataBuf, final RandomAccessFile FileHandle) throws IOException {
1162+
int Size = 257; // max size of string field + 1 byte for the position
1163+
final int Len;
1164+
final byte[] Data = new byte[Size];
1165+
byte[] Buf;
11431166

11441167
if (_UseMemoryMappedFile) {
11451168
Position = Position - _MapDataOffset; // position stored in BIN file is for full file, not just the mapped data segment, so need to minus
1146-
Size = _MapDataBuffer.get((int) Position); // use absolute offset to be thread-safe
1147-
11481169
try {
1149-
Bytes = new byte[Size];
1150-
Buf.position((int) Position + 1);
1151-
Buf.get(Bytes, 0, Size);
1170+
DataBuf.position((int) Position);
1171+
if (DataBuf.remaining() < Size) {
1172+
Size = DataBuf.remaining();
1173+
}
1174+
DataBuf.get(Data, 0, Size);
1175+
Len = Data[0];
1176+
1177+
Buf = new byte[Len];
1178+
System.arraycopy(Data, 1, Buf, (int) 0, Len);
1179+
11521180
} catch (NegativeArraySizeException e) {
11531181
return null;
11541182
}
1183+
11551184
} else {
1156-
RH.seek(Position);
1157-
Size = RH.read();
1185+
FileHandle.seek(Position);
11581186
try {
1159-
Bytes = new byte[Size];
1160-
RH.read(Bytes, 0, Size);
1187+
FileHandle.read(Data, 0, Size);
1188+
Len = Data[0];
1189+
1190+
Buf = new byte[Len];
1191+
System.arraycopy(Data, 1, Buf, (int) 0, Len);
1192+
11611193
} catch (NegativeArraySizeException e) {
11621194
return null;
11631195
}
11641196
}
11651197

1166-
return new String(Bytes);
1198+
return new String(Buf);
11671199
}
11681200

11691201
private BigInteger[] IP2No(String IP) throws UnknownHostException {

0 commit comments

Comments
 (0)