0.2.5

Author: danila-schelkov

.gitignore

@@ -0,0 +1,3 @@
+
+.idea/
+.idea/vcs.xml

.idea/vcs.xml

@@ -1,6 +1,6 @@
 ### SC Compression
 
-Version 0.2.4
+Version 0.2.5
 -
 
 ### Tools:

README.md

@@ -0,0 +1,3 @@
+__all__ = [
+    'compression'
+]

build/lib/sc_compression/__init__.py

@@ -0,0 +1,120 @@
+import lzma
+from hashlib import md5
+
+import lzham
+
+from sc_compression.utils.reader import Reader
+from sc_compression.utils.writer import Writer
+
+
+class Decompressor(Reader):
+    def __init__(self, buffer: bytes):
+        super().__init__(buffer, 'little')
+
+    def get_signature(self):
+        if self.buffer[:3] == b'\x5d\x00\x00':
+            return 'lzma'
+        elif self.buffer.startswith(b'SC'):
+            if len(self.buffer) >= 30 and self.buffer[26:30] == b'SCLZ':
+                return 'sclz'
+            return 'sc'
+        elif self.buffer[:4] == 'SIG:':
+            return 'sig'
+        return None
+
+    def decompress(self) -> bytes:
+        signature = self.get_signature()
+        if signature is None:
+            return self.buffer
+        elif signature == 'sc':
+            self.buffer = self.buffer[26:]
+            self.decompress()
+        elif signature == 'sig':
+            self.buffer = self.buffer[68:]
+            self.decompress()
+        elif signature == 'sclz':
+            self.read(30)
+            dict_size_log2 = self.readUByte()
+            uncompressed_size = self.readInt32()
+
+            filters = {
+                'dict_size_log2': dict_size_log2
+            }
+            decompressed = lzham.decompress(self.buffer[35:], uncompressed_size, filters)
+        elif signature == 'lzma':
+            decompressor = lzma.LZMADecompressor()
+
+            self.read(5)
+            uncompressed_size = self.readInt32()
+
+            lzma_byte = (b'\xff' if uncompressed_size == -1 else b'\x00')
+            compressed = self.buffer[:9] + lzma_byte * 4 + self.buffer[9:]
+
+            decompressed = decompressor.decompress(compressed)
+        else:
+            raise TypeError(signature)
+
+        return decompressed
+
+
+class Compressor(Writer):
+    lzham_filters = {
+        'dict_size_log2': 18
+    }
+    lzma_filters = [
+        {
+            "id": lzma.FILTER_LZMA1,
+            "dict_size": 256 * 1024,
+            "lc": 3,
+            "lp": 0,
+            "pb": 2,
+            "mode": lzma.MODE_NORMAL
+        },
+    ]
+
+    def __init__(self):
+        super().__init__('little')
+
+    def compress(self, data, signature: str) -> bytes:
+        uncompressed_size = len(data)
+
+        if signature is None:
+            return data
+        elif signature in ['lzma', 'sc', 'sig']:
+            compressor = lzma.LZMACompressor(format=lzma.FORMAT_ALONE, filters=self.lzma_filters)
+
+            compressed = compressor.compress(data)
+
+            self.write(compressed[:9])  # [:5]
+
+            # self.writeUInt32(uncompressed_size)
+
+            self.write(compressed[13:])
+
+            compressed = self.buffer
+        elif signature == 'sclz':
+            compressed = lzham.compress(data, filters=self.lzham_filters)
+
+            self.write(b'SCLZ')
+            self.writeUByte(18)
+            self.writeInt32(uncompressed_size)
+            self.write(compressed)
+
+            compressed = self.buffer
+        else:
+            raise TypeError('Unknown Signature: ' + signature)
+
+        super().__init__('big')
+        if signature in ['sc', 'sclz']:
+            data_hash = md5(data)
+
+            self.write(b'SC')
+            self.writeInt32(1)
+            self.writeInt32(16)
+            compressed = self.buffer + data_hash.digest() + compressed
+        elif signature == 'sig':
+            self.write(b'Sig:')
+            self.write(b'\x00'*64)  # sha64
+            compressed = self.buffer + compressed
+
+        return compressed

build/lib/sc_compression/compression.py

@@ -0,0 +1,4 @@
+__all__ = [
+    'reader',
+    'writer'
+]

build/lib/sc_compression/utils/__init__.py

@@ -0,0 +1,119 @@
+class Reader:
+    def __init__(self, buffer: bytes, endian: str = 'big'):
+        self.buffer = buffer
+        self.endian = endian
+        self.i = 0
+
+    def read(self, length: int = 1):
+        result = self.buffer[self.i:self.i + length]
+        self.i += length
+
+        return result
+
+    def readUInteger(self, length: int = 1) -> int:
+        result = 0
+        for x in range(length):
+            byte = self.buffer[self.i]
+
+            bit_padding = x * 8
+            if self.endian == 'big':
+                bit_padding = (8 * (length - 1)) - bit_padding
+
+            result |= byte << bit_padding
+            self.i += 1
+
+        return result
+
+    def readInteger(self, length: int = 1) -> int:
+        integer = self.readUInteger(length)
+        result = integer
+        if integer > 2**(length * 8) / 2:
+            result -= 2**(length * 8)
+        return result
+
+    def readUInt64(self) -> int:
+        return self.readUInteger(8)
+
+    def readInt64(self) -> int:
+        return self.readInteger(8)
+
+    def readFloat(self) -> float:
+        asInt = self.readUInt32()
+        binary = bin(asInt)
+        binary = binary[2:].zfill(32)
+
+        sign = -1 if binary[0] == 1 else 1
+        exponent = int(binary[1:9], 2) - 127
+        mantissa_base = binary[9:]
+        mantissa_bin = '1' + mantissa_base
+        mantissa = 0
+        val = 1
+
+        if exponent == -127:
+            if mantissa_base[1] == -1:
+                return 0
+            else:
+                exponent = -126
+                mantissa_bin = '0' + mantissa_base
+
+        for char in mantissa_bin:
+            mantissa += val * int(char)
+            val = val / 2
+
+        result = sign * 2 ** exponent * mantissa
+        return result
+
+    def readUInt32(self) -> int:
+        return self.readUInteger(4)
+
+    def readInt32(self) -> int:
+        return self.readInteger(4)
+
+    def readNUInt16(self) -> float:
+        return self.readUInt16() / 65535
+
+    def readUInt16(self) -> int:
+        return self.readUInteger(2)
+
+    def readNInt16(self) -> float:
+        return self.readInt16() / 32512
+
+    def readInt16(self) -> int:
+        return self.readInteger(2)
+
+    def readUInt8(self) -> int:
+        return self.readUInteger()
+
+    def readInt8(self) -> int:
+        return self.readInteger()
+
+    def readBool(self) -> bool:
+        if self.readUInt8() >= 1:
+            return True
+        else:
+            return False
+
+    readUInt = readUInteger
+    readInt = readInteger
+
+    readULong = readUInt64
+    readLong = readInt64
+
+    readNUShort = readNUInt16
+    readNShort = readNInt16
+
+    readUShort = readUInt16
+    readShort = readInt16
+
+    readUByte = readUInt8
+    readByte = readInt8
+
+    def readChar(self, length: int = 1) -> str:
+        return self.read(length).decode('utf-8')
+
+    def readString(self) -> str:
+        length = self.readUShort()
+        return self.readChar(length)
+
+    def tell(self) -> int:
+        return self.i

build/lib/sc_compression/utils/reader.py

@@ -0,0 +1,117 @@
+class Writer:
+    def __init__(self, endian: str = 'big'):
+        super(Writer, self).__init__()
+        self.endian = endian
+        self.buffer = b''
+
+    def write(self, data: bytes):
+        self.buffer += data
+
+    def writeUInteger(self, integer: int, length: int = 1):
+        self.buffer += integer.to_bytes(length, self.endian, signed=False)
+
+    def writeInteger(self, integer: int, length: int = 1):
+        self.buffer += integer.to_bytes(length, self.endian, signed=True)
+
+    def writeUInt64(self, integer: int):
+        self.writeUInteger(integer, 8)
+
+    def writeInt64(self, integer: int):
+        self.writeInteger(integer, 8)
+
+    def writeFloat(self, floating: float):
+        exponent = 0
+        sign = 1
+
+        if floating < 0:
+            sign = -1
+            floating = -floating
+
+        if floating >= 2 ** -1022:
+            value = floating
+
+            while value < 1:
+                exponent -= 1
+                value *= 2
+            while value >= 2:
+                exponent += 1
+                value /= 2
+
+        mantissa = floating / 2 ** exponent
+
+        exponent += 127
+
+        as_integer_bin = '0'
+        if sign == -1:
+            as_integer_bin = '1'
+
+        as_integer_bin += bin(exponent)[2:].zfill(8)
+
+        mantissa_bin = ''
+        for x in range(24):
+            bit = '0'
+            if mantissa >= 1/2**x:
+                mantissa -= 1/2**x
+                bit = '1'
+            mantissa_bin += bit
+
+        mantissa_bin = mantissa_bin[1:]
+
+        as_integer_bin += mantissa_bin
+        as_integer = int(as_integer_bin, 2)
+
+        self.writeUInt32(as_integer)
+
+    def writeUInt32(self, integer: int):
+        self.writeUInteger(integer, 4)
+
+    def writeInt32(self, integer: int):
+        self.writeInteger(integer, 4)
+
+    def writeNUInt16(self, integer: int):
+        self.writeUInt16(integer * 65535)
+
+    def writeUInt16(self, integer: int):
+        self.writeUInteger(integer, 2)
+
+    def writeNInt16(self, integer: int):
+        self.writeInt16(integer * 32512)
+
+    def writeInt16(self, integer: int):
+        self.writeInteger(integer, 2)
+
+    def writeUInt8(self, integer: int):
+        self.writeUInteger(integer)
+
+    def writeInt8(self, integer: int):
+        self.writeInteger(integer)
+
+    def writeBool(self, boolean: bool):
+        if boolean:
+            self.writeUInt8(1)
+        else:
+            self.writeUInt8(0)
+
+    writeUInt = writeUInteger
+    writeInt = writeInteger
+
+    writeULong = writeUInt64
+    writeLong = writeInt64
+
+    writeNUShort = writeNUInt16
+    writeNShort = writeNInt16
+
+    writeUShort = writeUInt16
+    writeShort = writeInt16
+
+    writeUByte = writeUInt8
+    writeByte = writeInt8
+
+    def writeChar(self, string: str):
+        for char in list(string):
+            self.buffer += char.encode('utf-8')
+
+    def writeString(self, string: str):
+        encoded = string.encode('utf-8')
+        self.writeUShort(len(encoded))
+        self.buffer += encoded