Skip to content

Commit c16db9b

Browse files
0.2.5
1 parent 54dbda1 commit c16db9b

File tree

18 files changed

+428
-14
lines changed

18 files changed

+428
-14
lines changed

.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
2+
.idea/
3+
.idea/vcs.xml

.idea/vcs.xml

Lines changed: 6 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
### SC Compression
22

3-
Version 0.2.4
3+
Version 0.2.5
44
-
55

66
### Tools:

build/lib/sc_compression/__init__.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
__all__ = [
2+
'compression'
3+
]
Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
import lzma
2+
from hashlib import md5
3+
4+
import lzham
5+
6+
from sc_compression.utils.reader import Reader
7+
from sc_compression.utils.writer import Writer
8+
9+
10+
class Decompressor(Reader):
11+
def __init__(self, buffer: bytes):
12+
super().__init__(buffer, 'little')
13+
14+
def get_signature(self):
15+
if self.buffer[:3] == b'\x5d\x00\x00':
16+
return 'lzma'
17+
elif self.buffer.startswith(b'SC'):
18+
if len(self.buffer) >= 30 and self.buffer[26:30] == b'SCLZ':
19+
return 'sclz'
20+
return 'sc'
21+
elif self.buffer[:4] == 'SIG:':
22+
return 'sig'
23+
return None
24+
25+
def decompress(self) -> bytes:
26+
signature = self.get_signature()
27+
if signature is None:
28+
return self.buffer
29+
elif signature == 'sc':
30+
self.buffer = self.buffer[26:]
31+
self.decompress()
32+
elif signature == 'sig':
33+
self.buffer = self.buffer[68:]
34+
self.decompress()
35+
elif signature == 'sclz':
36+
self.read(30)
37+
dict_size_log2 = self.readUByte()
38+
uncompressed_size = self.readInt32()
39+
40+
filters = {
41+
'dict_size_log2': dict_size_log2
42+
}
43+
decompressed = lzham.decompress(self.buffer[35:], uncompressed_size, filters)
44+
elif signature == 'lzma':
45+
decompressor = lzma.LZMADecompressor()
46+
47+
self.read(5)
48+
uncompressed_size = self.readInt32()
49+
50+
lzma_byte = (b'\xff' if uncompressed_size == -1 else b'\x00')
51+
compressed = self.buffer[:9] + lzma_byte * 4 + self.buffer[9:]
52+
53+
decompressed = decompressor.decompress(compressed)
54+
else:
55+
raise TypeError(signature)
56+
57+
return decompressed
58+
59+
60+
class Compressor(Writer):
61+
lzham_filters = {
62+
'dict_size_log2': 18
63+
}
64+
lzma_filters = [
65+
{
66+
"id": lzma.FILTER_LZMA1,
67+
"dict_size": 256 * 1024,
68+
"lc": 3,
69+
"lp": 0,
70+
"pb": 2,
71+
"mode": lzma.MODE_NORMAL
72+
},
73+
]
74+
75+
def __init__(self):
76+
super().__init__('little')
77+
78+
def compress(self, data, signature: str) -> bytes:
79+
uncompressed_size = len(data)
80+
81+
if signature is None:
82+
return data
83+
elif signature in ['lzma', 'sc', 'sig']:
84+
compressor = lzma.LZMACompressor(format=lzma.FORMAT_ALONE, filters=self.lzma_filters)
85+
86+
compressed = compressor.compress(data)
87+
88+
self.write(compressed[:9]) # [:5]
89+
90+
# self.writeUInt32(uncompressed_size)
91+
92+
self.write(compressed[13:])
93+
94+
compressed = self.buffer
95+
elif signature == 'sclz':
96+
compressed = lzham.compress(data, filters=self.lzham_filters)
97+
98+
self.write(b'SCLZ')
99+
self.writeUByte(18)
100+
self.writeInt32(uncompressed_size)
101+
self.write(compressed)
102+
103+
compressed = self.buffer
104+
else:
105+
raise TypeError('Unknown Signature: ' + signature)
106+
107+
super().__init__('big')
108+
if signature in ['sc', 'sclz']:
109+
data_hash = md5(data)
110+
111+
self.write(b'SC')
112+
self.writeInt32(1)
113+
self.writeInt32(16)
114+
compressed = self.buffer + data_hash.digest() + compressed
115+
elif signature == 'sig':
116+
self.write(b'Sig:')
117+
self.write(b'\x00'*64) # sha64
118+
compressed = self.buffer + compressed
119+
120+
return compressed
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
__all__ = [
2+
'reader',
3+
'writer'
4+
]
Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
class Reader:
2+
def __init__(self, buffer: bytes, endian: str = 'big'):
3+
self.buffer = buffer
4+
self.endian = endian
5+
self.i = 0
6+
7+
def read(self, length: int = 1):
8+
result = self.buffer[self.i:self.i + length]
9+
self.i += length
10+
11+
return result
12+
13+
def readUInteger(self, length: int = 1) -> int:
14+
result = 0
15+
for x in range(length):
16+
byte = self.buffer[self.i]
17+
18+
bit_padding = x * 8
19+
if self.endian == 'big':
20+
bit_padding = (8 * (length - 1)) - bit_padding
21+
22+
result |= byte << bit_padding
23+
self.i += 1
24+
25+
return result
26+
27+
def readInteger(self, length: int = 1) -> int:
28+
integer = self.readUInteger(length)
29+
result = integer
30+
if integer > 2**(length * 8) / 2:
31+
result -= 2**(length * 8)
32+
return result
33+
34+
def readUInt64(self) -> int:
35+
return self.readUInteger(8)
36+
37+
def readInt64(self) -> int:
38+
return self.readInteger(8)
39+
40+
def readFloat(self) -> float:
41+
asInt = self.readUInt32()
42+
binary = bin(asInt)
43+
binary = binary[2:].zfill(32)
44+
45+
sign = -1 if binary[0] == 1 else 1
46+
exponent = int(binary[1:9], 2) - 127
47+
mantissa_base = binary[9:]
48+
mantissa_bin = '1' + mantissa_base
49+
mantissa = 0
50+
val = 1
51+
52+
if exponent == -127:
53+
if mantissa_base[1] == -1:
54+
return 0
55+
else:
56+
exponent = -126
57+
mantissa_bin = '0' + mantissa_base
58+
59+
for char in mantissa_bin:
60+
mantissa += val * int(char)
61+
val = val / 2
62+
63+
result = sign * 2 ** exponent * mantissa
64+
return result
65+
66+
def readUInt32(self) -> int:
67+
return self.readUInteger(4)
68+
69+
def readInt32(self) -> int:
70+
return self.readInteger(4)
71+
72+
def readNUInt16(self) -> float:
73+
return self.readUInt16() / 65535
74+
75+
def readUInt16(self) -> int:
76+
return self.readUInteger(2)
77+
78+
def readNInt16(self) -> float:
79+
return self.readInt16() / 32512
80+
81+
def readInt16(self) -> int:
82+
return self.readInteger(2)
83+
84+
def readUInt8(self) -> int:
85+
return self.readUInteger()
86+
87+
def readInt8(self) -> int:
88+
return self.readInteger()
89+
90+
def readBool(self) -> bool:
91+
if self.readUInt8() >= 1:
92+
return True
93+
else:
94+
return False
95+
96+
readUInt = readUInteger
97+
readInt = readInteger
98+
99+
readULong = readUInt64
100+
readLong = readInt64
101+
102+
readNUShort = readNUInt16
103+
readNShort = readNInt16
104+
105+
readUShort = readUInt16
106+
readShort = readInt16
107+
108+
readUByte = readUInt8
109+
readByte = readInt8
110+
111+
def readChar(self, length: int = 1) -> str:
112+
return self.read(length).decode('utf-8')
113+
114+
def readString(self) -> str:
115+
length = self.readUShort()
116+
return self.readChar(length)
117+
118+
def tell(self) -> int:
119+
return self.i
Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
class Writer:
2+
def __init__(self, endian: str = 'big'):
3+
super(Writer, self).__init__()
4+
self.endian = endian
5+
self.buffer = b''
6+
7+
def write(self, data: bytes):
8+
self.buffer += data
9+
10+
def writeUInteger(self, integer: int, length: int = 1):
11+
self.buffer += integer.to_bytes(length, self.endian, signed=False)
12+
13+
def writeInteger(self, integer: int, length: int = 1):
14+
self.buffer += integer.to_bytes(length, self.endian, signed=True)
15+
16+
def writeUInt64(self, integer: int):
17+
self.writeUInteger(integer, 8)
18+
19+
def writeInt64(self, integer: int):
20+
self.writeInteger(integer, 8)
21+
22+
def writeFloat(self, floating: float):
23+
exponent = 0
24+
sign = 1
25+
26+
if floating < 0:
27+
sign = -1
28+
floating = -floating
29+
30+
if floating >= 2 ** -1022:
31+
value = floating
32+
33+
while value < 1:
34+
exponent -= 1
35+
value *= 2
36+
while value >= 2:
37+
exponent += 1
38+
value /= 2
39+
40+
mantissa = floating / 2 ** exponent
41+
42+
exponent += 127
43+
44+
as_integer_bin = '0'
45+
if sign == -1:
46+
as_integer_bin = '1'
47+
48+
as_integer_bin += bin(exponent)[2:].zfill(8)
49+
50+
mantissa_bin = ''
51+
for x in range(24):
52+
bit = '0'
53+
if mantissa >= 1/2**x:
54+
mantissa -= 1/2**x
55+
bit = '1'
56+
mantissa_bin += bit
57+
58+
mantissa_bin = mantissa_bin[1:]
59+
60+
as_integer_bin += mantissa_bin
61+
as_integer = int(as_integer_bin, 2)
62+
63+
self.writeUInt32(as_integer)
64+
65+
def writeUInt32(self, integer: int):
66+
self.writeUInteger(integer, 4)
67+
68+
def writeInt32(self, integer: int):
69+
self.writeInteger(integer, 4)
70+
71+
def writeNUInt16(self, integer: int):
72+
self.writeUInt16(integer * 65535)
73+
74+
def writeUInt16(self, integer: int):
75+
self.writeUInteger(integer, 2)
76+
77+
def writeNInt16(self, integer: int):
78+
self.writeInt16(integer * 32512)
79+
80+
def writeInt16(self, integer: int):
81+
self.writeInteger(integer, 2)
82+
83+
def writeUInt8(self, integer: int):
84+
self.writeUInteger(integer)
85+
86+
def writeInt8(self, integer: int):
87+
self.writeInteger(integer)
88+
89+
def writeBool(self, boolean: bool):
90+
if boolean:
91+
self.writeUInt8(1)
92+
else:
93+
self.writeUInt8(0)
94+
95+
writeUInt = writeUInteger
96+
writeInt = writeInteger
97+
98+
writeULong = writeUInt64
99+
writeLong = writeInt64
100+
101+
writeNUShort = writeNUInt16
102+
writeNShort = writeNInt16
103+
104+
writeUShort = writeUInt16
105+
writeShort = writeInt16
106+
107+
writeUByte = writeUInt8
108+
writeByte = writeInt8
109+
110+
def writeChar(self, string: str):
111+
for char in list(string):
112+
self.buffer += char.encode('utf-8')
113+
114+
def writeString(self, string: str):
115+
encoded = string.encode('utf-8')
116+
self.writeUShort(len(encoded))
117+
self.buffer += encoded

dist/sc-compression-0.2.5.tar.gz

3.67 KB
Binary file not shown.
16.8 KB
Binary file not shown.

0 commit comments

Comments
 (0)