diff --git a/.gitignore b/.gitignore index 3d798f0..94ff474 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,7 @@ __pycache__ *.pyc *.egg-info .vscode -.coverage \ No newline at end of file +.coverage +data.txt +test_input.txt +*.tmp \ No newline at end of file diff --git a/src/ur/file_fountain_decoder.py b/src/ur/file_fountain_decoder.py new file mode 100644 index 0000000..9d22a13 --- /dev/null +++ b/src/ur/file_fountain_decoder.py @@ -0,0 +1,170 @@ +# +# Copyright © 2020 Foundation Devices, Inc. and Contributors +# Licensed under the "BSD-2-Clause Plus Patent License" +# + +import os +import binascii +from .fountain_decoder import FountainDecoder, InvalidChecksum +from .crc32 import crc32 + + +class FileFountainDecoder(FountainDecoder): + """ + Fountain decoder that uses a files for fragments to minimize RAM usage. + """ + + # optimizations + MAX_MIXED_XOR_DEGREE = 3 + MAX_MIXED_INDEXES = 6 + LEN_SIMPLE_AVOID_MIX = 60 + + def __init__(self, workdir): + super().__init__() + self.mixed_part_indexes = set() + self.workdir = workdir + self._clear_files() + + # optimization for MAX_MIXED_INDEXES + def _limit_mixed_set(self, mix_set): + if len(mix_set) > FileFountainDecoder.MAX_MIXED_INDEXES: + best = sorted(mix_set, key=len)[: FileFountainDecoder.MAX_MIXED_INDEXES] + mix_set.clear() + mix_set.update(best) + + def _fragment_path(self, index): + return self.workdir + "/" + "%s.tmp" % index + + def _encode_indexes(self, indexes): + b = bytes(indexes) + s = binascii.b2a_base64(b) + return s.replace(b"+", b"-").replace(b"/", b"_").replace(b"\n", b"").decode() + + def _fragment_path_mixed(self, indexes): + return self._fragment_path(self._encode_indexes(indexes)) + + def _clear_files(self): + for name in os.listdir(self.workdir): + if name.endswith(".tmp"): + os.remove(self.workdir + "/" + name) + + # overrides ------ + + def _clear_caches(self): + super()._clear_caches() + self.mixed_part_indexes.clear() + self._clear_files() + + def join_fragments_from_files(self, out_path): + remaining = self.expected_message_len + checksum = 0 + + buf = bytearray(self.expected_fragment_len) + mv = memoryview(buf) + + with open(out_path, "wb") as out: + for i in range(self.expected_part_count()): + if remaining <= 0: + break + + with open(self._fragment_path(i), "rb") as frag: + while remaining > 0: + n = frag.readinto(buf) + if n == 0: + break + + # pylint: disable=consider-using-min-builtin + if n > remaining: + n = remaining + + chunk = mv[:n] + checksum = crc32(chunk, checksum) + out.write(chunk) + remaining -= n + + return checksum + + def _store_fragment(self, index, part): + filename = ( + self._fragment_path(index) + if isinstance(index, int) + else self._fragment_path_mixed(index) + ) + with open(filename, "wb") as f: + f.write(part.data) + + def _finalize_message(self): + out_path = self.workdir + "/" + "data.txt" + + checksum = self.join_fragments_from_files(out_path) + + if checksum == self.expected_checksum: + self.result = out_path # or open file handle if you prefer + else: + self.result = InvalidChecksum() + + def _retrieve_part_data(self, p): + if isinstance(p.data, str): + with open(p.data, "rb") as frag: + p.data = frag.read() + + def reduce_mixed_by(self, p): + # avoid looking for mixed if already got many simple parts + if len(self.received_part_indexes) > FileFountainDecoder.LEN_SIMPLE_AVOID_MIX: + return + + new_mixed_indexes = set() + for indexes in self.mixed_part_indexes: + # avoid XOR with this mixed parts + if len(indexes) > FileFountainDecoder.MAX_MIXED_XOR_DEGREE: + new_mixed_indexes.add(indexes) + continue + + value_bytearray = bytearray(self.expected_fragment_len) + with open(self._fragment_path_mixed(indexes), "rb") as mixed_frag: + mixed_frag.readinto(value_bytearray) + reduced = self.reduced_part_by_part( + FountainDecoder.Part(indexes, value_bytearray), p + ) + if reduced.is_simple(): + self.queued_parts.append(reduced) + else: + self._store_fragment(reduced.indexes, reduced) + new_mixed_indexes.add(reduced.indexes) + + self._limit_mixed_set(new_mixed_indexes) + self.mixed_part_indexes.clear() + self.mixed_part_indexes.update(new_mixed_indexes) + + def process_mixed_part(self, p): + # Don't process duplicate parts + if p.indexes in self.mixed_part_indexes: + return + + # Reduce this part by all the others + reduced = p + + for index in self.received_part_indexes: + r = FountainDecoder.Part(frozenset([index]), self._fragment_path(index)) + reduced = self.reduced_part_by_part(reduced, r) + if reduced.is_simple(): + break + + if not reduced.is_simple(): + for indexes in self.mixed_part_indexes: + r = FountainDecoder.Part(indexes, self._fragment_path_mixed(indexes)) + reduced = self.reduced_part_by_part(reduced, r) + if reduced.is_simple(): + break + + # If the part is now simple + if reduced.is_simple(): + # Add it to the queue + self.queued_parts.append(reduced) + else: + # Reduce all the mixed parts by this one + self.reduce_mixed_by(reduced) + # Record this new mixed part + self._store_fragment(reduced.indexes, reduced) + self.mixed_part_indexes.add(reduced.indexes) + self._limit_mixed_set(self.mixed_part_indexes) diff --git a/src/ur/file_fountain_encoder.py b/src/ur/file_fountain_encoder.py new file mode 100644 index 0000000..efa8aa7 --- /dev/null +++ b/src/ur/file_fountain_encoder.py @@ -0,0 +1,80 @@ +# +# Copyright © 2020 Foundation Devices, Inc. and Contributors +# Licensed under the "BSD-2-Clause Plus Patent License" +# + +import os +from .fountain_encoder import FountainEncoder +from .utils import xor_into +from .constants import MAX_UINT32 +from .crc32 import crc32 + + +class FileFountainEncoder(FountainEncoder): + # pylint: disable=super-init-not-called + def __init__( + self, + file_path, + max_fragment_len, + first_seq_num=0, + min_fragment_len=10, + ): + self.file_path = file_path + + self.message_len = os.stat(file_path)[6] + assert self.message_len <= MAX_UINT32 + + self.checksum = self._compute_checksum() + + self.fragment_len = self.find_nominal_fragment_length( + self.message_len, + min_fragment_len, + max_fragment_len, + ) + + self.seq_num = first_seq_num + + def _read_range(self, offset, length): + with open(self.file_path, "rb") as f: + f.seek(offset) + return f.read(length) + + def _compute_checksum(self): + checksum = 0 + remaining = self.message_len + + buf = bytearray(256) + mv = memoryview(buf) + + with open(self.file_path, "rb") as f: + while remaining > 0: + n = f.readinto(buf) + if n == 0: + break + + # pylint: disable=consider-using-min-builtin + if n > remaining: + n = remaining + + checksum = crc32(mv[:n], checksum) + remaining -= n + + return checksum + + # overrides ------ + + # XOR selected fragments + def mix(self, indexes): + result = bytearray(self.fragment_len) + frag_len = self.fragment_len + msg_len = self.message_len + + for index in indexes: + start = index * frag_len + if start >= msg_len: + continue + + size = min(frag_len, msg_len - start) + xor_into(result, self._read_range(start, size)) + + return result diff --git a/src/ur/file_ur_decoder.py b/src/ur/file_ur_decoder.py new file mode 100644 index 0000000..e3b8ec5 --- /dev/null +++ b/src/ur/file_ur_decoder.py @@ -0,0 +1,17 @@ +# +# Copyright © 2020 Foundation Devices, Inc. and Contributors +# Licensed under the "BSD-2-Clause Plus Patent License" +# + +from .ur_decoder import URDecoder +from .file_fountain_decoder import FileFountainDecoder + + +class FileURDecoder(URDecoder): + """ + UR decoder that uses a FileFountainDecoder to minimize RAM usage. + """ + + def __init__(self, workdir): + super().__init__() + self.fountain_decoder = FileFountainDecoder(workdir) diff --git a/src/ur/file_ur_encoder.py b/src/ur/file_ur_encoder.py new file mode 100644 index 0000000..b5e1e2f --- /dev/null +++ b/src/ur/file_ur_encoder.py @@ -0,0 +1,29 @@ +# +# Copyright © 2020 Foundation Devices, Inc. and Contributors +# Licensed under the "BSD-2-Clause Plus Patent License" +# + +from .ur_encoder import UREncoder +from .file_fountain_encoder import FileFountainEncoder + + +class FileUREncoder(UREncoder): + """ + UR decoder that uses a FileFountainEncoder to minimize RAM usage. + """ + + # pylint: disable=super-init-not-called + def __init__(self, ur, max_fragment_len, first_seq_num=0, min_fragment_len=10): + self.ur = ur + self.fountain_encoder = FileFountainEncoder( + ur.cbor, max_fragment_len, first_seq_num, min_fragment_len + ) + + def next_part(self): + part = self.fountain_encoder.next_part() + if self.is_single_part(): + with open(self.fountain_encoder.file_path, "rb") as f: + self.ur.cbor = f.read() + return UREncoder.encode(self.ur) + + return UREncoder.encode_part(self.ur.type, part) diff --git a/src/ur/fountain_decoder.py b/src/ur/fountain_decoder.py index 519170e..37ffc8b 100644 --- a/src/ur/fountain_decoder.py +++ b/src/ur/fountain_decoder.py @@ -164,7 +164,8 @@ def reduced_part_by_part(self, a, b): # `a` is not reducable by `b`, so return a return a - def _store_fragment(self, _index, part): + # pylint: disable=unused-argument + def _store_fragment(self, index, part): # store whole part self.simple_parts[part.indexes] = part diff --git a/test.py b/test.py index 71205d1..4b64520 100644 --- a/test.py +++ b/test.py @@ -10,6 +10,7 @@ except: unittest = None +from unittest import mock from test_utils import make_message, make_message_ur, next_data # from ur.bytewords import STYLE_STANDARD, STYLE_URI, STYLE_MINIMAL @@ -24,6 +25,76 @@ from ur.ur_decoder import URDecoder from ur.crc32 import crc32 +TMP_FILE = "./test_input.txt" + +FOUNTAIN_ENCODER_EXPECTED_PARTS = [ + "seqNum:1, seqLen:9, messageLen:256, checksum:23570951, data:916ec65cf77cadf55cd7f9cda1a1030026ddd42e905b77adc36e4f2d3c", + "seqNum:2, seqLen:9, messageLen:256, checksum:23570951, data:cba44f7f04f2de44f42d84c374a0e149136f25b01852545961d55f7f7a", + "seqNum:3, seqLen:9, messageLen:256, checksum:23570951, data:8cde6d0e2ec43f3b2dcb644a2209e8c9e34af5c4747984a5e873c9cf5f", + "seqNum:4, seqLen:9, messageLen:256, checksum:23570951, data:965e25ee29039fdf8ca74f1c769fc07eb7ebaec46e0695aea6cbd60b3e", + "seqNum:5, seqLen:9, messageLen:256, checksum:23570951, data:c4bbff1b9ffe8a9e7240129377b9d3711ed38d412fbb4442256f1e6f59", + "seqNum:6, seqLen:9, messageLen:256, checksum:23570951, data:5e0fc57fed451fb0a0101fb76b1fb1e1b88cfdfdaa946294a47de8fff1", + "seqNum:7, seqLen:9, messageLen:256, checksum:23570951, data:73f021c0e6f65b05c0a494e50791270a0050a73ae69b6725505a2ec8a5", + "seqNum:8, seqLen:9, messageLen:256, checksum:23570951, data:791457c9876dd34aadd192a53aa0dc66b556c0c215c7ceb8248b717c22", + "seqNum:9, seqLen:9, messageLen:256, checksum:23570951, data:951e65305b56a3706e3e86eb01c803bbf915d80edcd64d4d0000000000", + "seqNum:10, seqLen:9, messageLen:256, checksum:23570951, data:330f0f33a05eead4f331df229871bee733b50de71afd2e5a79f196de09", + "seqNum:11, seqLen:9, messageLen:256, checksum:23570951, data:3b205ce5e52d8c24a52cffa34c564fa1af3fdffcd349dc4258ee4ee828", + "seqNum:12, seqLen:9, messageLen:256, checksum:23570951, data:dd7bf725ea6c16d531b5f03254783803048ca08b87148daacd1cd7a006", + "seqNum:13, seqLen:9, messageLen:256, checksum:23570951, data:760be7ad1c6187902bbc04f539b9ee5eb8ea6833222edea36031306c01", + "seqNum:14, seqLen:9, messageLen:256, checksum:23570951, data:5bf4031217d2c3254b088fa7553778b5003632f46e21db129416f65b55", + "seqNum:15, seqLen:9, messageLen:256, checksum:23570951, data:73f021c0e6f65b05c0a494e50791270a0050a73ae69b6725505a2ec8a5", + "seqNum:16, seqLen:9, messageLen:256, checksum:23570951, data:b8546ebfe2048541348910267331c643133f828afec9337c318f71b7df", + "seqNum:17, seqLen:9, messageLen:256, checksum:23570951, data:23dedeea74e3a0fb052befabefa13e2f80e4315c9dceed4c8630612e64", + "seqNum:18, seqLen:9, messageLen:256, checksum:23570951, data:d01a8daee769ce34b6b35d3ca0005302724abddae405bdb419c0a6b208", + "seqNum:19, seqLen:9, messageLen:256, checksum:23570951, data:3171c5dc365766eff25ae47c6f10e7de48cfb8474e050e5fe997a6dc24", + "seqNum:20, seqLen:9, messageLen:256, checksum:23570951, data:e055c2433562184fa71b4be94f262e200f01c6f74c284b0dc6fae6673f" + ] + +FOUNTAIN_ENCODER_CBOR_EXPECTED_PARTS = [ + "8501091901001a0167aa07581d916ec65cf77cadf55cd7f9cda1a1030026ddd42e905b77adc36e4f2d3c", + "8502091901001a0167aa07581dcba44f7f04f2de44f42d84c374a0e149136f25b01852545961d55f7f7a", + "8503091901001a0167aa07581d8cde6d0e2ec43f3b2dcb644a2209e8c9e34af5c4747984a5e873c9cf5f", + "8504091901001a0167aa07581d965e25ee29039fdf8ca74f1c769fc07eb7ebaec46e0695aea6cbd60b3e", + "8505091901001a0167aa07581dc4bbff1b9ffe8a9e7240129377b9d3711ed38d412fbb4442256f1e6f59", + "8506091901001a0167aa07581d5e0fc57fed451fb0a0101fb76b1fb1e1b88cfdfdaa946294a47de8fff1", + "8507091901001a0167aa07581d73f021c0e6f65b05c0a494e50791270a0050a73ae69b6725505a2ec8a5", + "8508091901001a0167aa07581d791457c9876dd34aadd192a53aa0dc66b556c0c215c7ceb8248b717c22", + "8509091901001a0167aa07581d951e65305b56a3706e3e86eb01c803bbf915d80edcd64d4d0000000000", + "850a091901001a0167aa07581d330f0f33a05eead4f331df229871bee733b50de71afd2e5a79f196de09", + "850b091901001a0167aa07581d3b205ce5e52d8c24a52cffa34c564fa1af3fdffcd349dc4258ee4ee828", + "850c091901001a0167aa07581ddd7bf725ea6c16d531b5f03254783803048ca08b87148daacd1cd7a006", + "850d091901001a0167aa07581d760be7ad1c6187902bbc04f539b9ee5eb8ea6833222edea36031306c01", + "850e091901001a0167aa07581d5bf4031217d2c3254b088fa7553778b5003632f46e21db129416f65b55", + "850f091901001a0167aa07581d73f021c0e6f65b05c0a494e50791270a0050a73ae69b6725505a2ec8a5", + "8510091901001a0167aa07581db8546ebfe2048541348910267331c643133f828afec9337c318f71b7df", + "8511091901001a0167aa07581d23dedeea74e3a0fb052befabefa13e2f80e4315c9dceed4c8630612e64", + "8512091901001a0167aa07581dd01a8daee769ce34b6b35d3ca0005302724abddae405bdb419c0a6b208", + "8513091901001a0167aa07581d3171c5dc365766eff25ae47c6f10e7de48cfb8474e050e5fe997a6dc24", + "8514091901001a0167aa07581de055c2433562184fa71b4be94f262e200f01c6f74c284b0dc6fae6673f" + ] + +UR_ENCODER_EXPECTED_PARTS = [ + "ur:bytes/1-9/lpadascfadaxcywenbpljkhdcahkadaemejtswhhylkepmykhhtsytsnoyoyaxaedsuttydmmhhpktpmsrjtdkgslpgh", + "ur:bytes/2-9/lpaoascfadaxcywenbpljkhdcagwdpfnsboxgwlbaawzuefywkdplrsrjynbvygabwjldapfcsgmghhkhstlrdcxaefz", + "ur:bytes/3-9/lpaxascfadaxcywenbpljkhdcahelbknlkuejnbadmssfhfrdpsbiegecpasvssovlgeykssjykklronvsjksopdzmol", + "ur:bytes/4-9/lpaaascfadaxcywenbpljkhdcasotkhemthydawydtaxneurlkosgwcekonertkbrlwmplssjtammdplolsbrdzcrtas", + "ur:bytes/5-9/lpahascfadaxcywenbpljkhdcatbbdfmssrkzmcwnezelennjpfzbgmuktrhtejscktelgfpdlrkfyfwdajldejokbwf", + "ur:bytes/6-9/lpamascfadaxcywenbpljkhdcackjlhkhybssklbwefectpfnbbectrljectpavyrolkzczcpkmwidmwoxkilghdsowp", + "ur:bytes/7-9/lpatascfadaxcywenbpljkhdcavszmwnjkwtclrtvaynhpahrtoxmwvwatmedibkaegdosftvandiodagdhthtrlnnhy", + "ur:bytes/8-9/lpayascfadaxcywenbpljkhdcadmsponkkbbhgsoltjntegepmttmoonftnbuoiyrehfrtsabzsttorodklubbuyaetk", + "ur:bytes/9-9/lpasascfadaxcywenbpljkhdcajskecpmdckihdyhphfotjojtfmlnwmadspaxrkytbztpbauotbgtgtaeaevtgavtny", + "ur:bytes/10-9/lpbkascfadaxcywenbpljkhdcahkadaemejtswhhylkepmykhhtsytsnoyoyaxaedsuttydmmhhpktpmsrjtwdkiplzs", + "ur:bytes/11-9/lpbdascfadaxcywenbpljkhdcahelbknlkuejnbadmssfhfrdpsbiegecpasvssovlgeykssjykklronvsjkvetiiapk", + "ur:bytes/12-9/lpbnascfadaxcywenbpljkhdcarllaluzmdmgstospeyiefmwejlwtpedamktksrvlcygmzemovovllarodtmtbnptrs", + "ur:bytes/13-9/lpbtascfadaxcywenbpljkhdcamtkgtpknghchchyketwsvwgwfdhpgmgtylctotzopdrpayoschcmhplffziachrfgd", + "ur:bytes/14-9/lpbaascfadaxcywenbpljkhdcapazewnvonnvdnsbyleynwtnsjkjndeoldydkbkdslgjkbbkortbelomueekgvstegt", + "ur:bytes/15-9/lpbsascfadaxcywenbpljkhdcaynmhpddpzmversbdqdfyrehnqzlugmjzmnmtwmrouohtstgsbsahpawkditkckynwt", + "ur:bytes/16-9/lpbeascfadaxcywenbpljkhdcawygekobamwtlihsnpalnsghenskkiynthdzotsimtojetprsttmukirlrsbtamjtpd", + "ur:bytes/17-9/lpbyascfadaxcywenbpljkhdcamklgftaxykpewyrtqzhydntpnytyisincxmhtbceaykolduortotiaiaiafhiaoyce", + "ur:bytes/18-9/lpbgascfadaxcywenbpljkhdcahkadaemejtswhhylkepmykhhtsytsnoyoyaxaedsuttydmmhhpktpmsrjtntwkbkwy", + "ur:bytes/19-9/lpbwascfadaxcywenbpljkhdcadekicpaajootjzpsdrbalpeywllbdsnbinaerkurspbncxgslgftvtsrjtksplcpeo", + "ur:bytes/20-9/lpbbascfadaxcywenbpljkhdcayapmrleeleaxpasfrtrdkncffwjyjzgyetdmlewtkpktgllepfrltataztksmhkbot" + ] def data_to_hex(buf): return "".join("{:02x}".format(x) for x in buf) @@ -291,28 +362,33 @@ def description(part): for i in range(20): parts.append(description(encoder.next_part())) - expected_parts = [ - "seqNum:1, seqLen:9, messageLen:256, checksum:23570951, data:916ec65cf77cadf55cd7f9cda1a1030026ddd42e905b77adc36e4f2d3c", - "seqNum:2, seqLen:9, messageLen:256, checksum:23570951, data:cba44f7f04f2de44f42d84c374a0e149136f25b01852545961d55f7f7a", - "seqNum:3, seqLen:9, messageLen:256, checksum:23570951, data:8cde6d0e2ec43f3b2dcb644a2209e8c9e34af5c4747984a5e873c9cf5f", - "seqNum:4, seqLen:9, messageLen:256, checksum:23570951, data:965e25ee29039fdf8ca74f1c769fc07eb7ebaec46e0695aea6cbd60b3e", - "seqNum:5, seqLen:9, messageLen:256, checksum:23570951, data:c4bbff1b9ffe8a9e7240129377b9d3711ed38d412fbb4442256f1e6f59", - "seqNum:6, seqLen:9, messageLen:256, checksum:23570951, data:5e0fc57fed451fb0a0101fb76b1fb1e1b88cfdfdaa946294a47de8fff1", - "seqNum:7, seqLen:9, messageLen:256, checksum:23570951, data:73f021c0e6f65b05c0a494e50791270a0050a73ae69b6725505a2ec8a5", - "seqNum:8, seqLen:9, messageLen:256, checksum:23570951, data:791457c9876dd34aadd192a53aa0dc66b556c0c215c7ceb8248b717c22", - "seqNum:9, seqLen:9, messageLen:256, checksum:23570951, data:951e65305b56a3706e3e86eb01c803bbf915d80edcd64d4d0000000000", - "seqNum:10, seqLen:9, messageLen:256, checksum:23570951, data:330f0f33a05eead4f331df229871bee733b50de71afd2e5a79f196de09", - "seqNum:11, seqLen:9, messageLen:256, checksum:23570951, data:3b205ce5e52d8c24a52cffa34c564fa1af3fdffcd349dc4258ee4ee828", - "seqNum:12, seqLen:9, messageLen:256, checksum:23570951, data:dd7bf725ea6c16d531b5f03254783803048ca08b87148daacd1cd7a006", - "seqNum:13, seqLen:9, messageLen:256, checksum:23570951, data:760be7ad1c6187902bbc04f539b9ee5eb8ea6833222edea36031306c01", - "seqNum:14, seqLen:9, messageLen:256, checksum:23570951, data:5bf4031217d2c3254b088fa7553778b5003632f46e21db129416f65b55", - "seqNum:15, seqLen:9, messageLen:256, checksum:23570951, data:73f021c0e6f65b05c0a494e50791270a0050a73ae69b6725505a2ec8a5", - "seqNum:16, seqLen:9, messageLen:256, checksum:23570951, data:b8546ebfe2048541348910267331c643133f828afec9337c318f71b7df", - "seqNum:17, seqLen:9, messageLen:256, checksum:23570951, data:23dedeea74e3a0fb052befabefa13e2f80e4315c9dceed4c8630612e64", - "seqNum:18, seqLen:9, messageLen:256, checksum:23570951, data:d01a8daee769ce34b6b35d3ca0005302724abddae405bdb419c0a6b208", - "seqNum:19, seqLen:9, messageLen:256, checksum:23570951, data:3171c5dc365766eff25ae47c6f10e7de48cfb8474e050e5fe997a6dc24", - "seqNum:20, seqLen:9, messageLen:256, checksum:23570951, data:e055c2433562184fa71b4be94f262e200f01c6f74c284b0dc6fae6673f" - ] + expected_parts = FOUNTAIN_ENCODER_EXPECTED_PARTS + assert(parts == expected_parts) + + def test_file_fountain_encoder(self): + from ur.file_fountain_encoder import FileFountainEncoder + + message = make_message(256) + tmp_file_path = TMP_FILE + with open(tmp_file_path, "wb") as f: + f.write(message) + + encoder = FileFountainEncoder(tmp_file_path, 30) + parts = [] + + def description(part): + return "seqNum:{}, seqLen:{}, messageLen:{}, checksum:{}, data:{}".format( + part.seq_num, + part.seq_len, + part.message_len, + part.checksum, + data_to_hex(part.data), + ) + + for i in range(20): + parts.append(description(encoder.next_part())) + + expected_parts = FOUNTAIN_ENCODER_EXPECTED_PARTS assert(parts == expected_parts) @@ -323,28 +399,23 @@ def test_fountain_encoder_cbor(self): for i in range(20): parts.append(data_to_hex(encoder.next_part().cbor())) - expected_parts = [ - "8501091901001a0167aa07581d916ec65cf77cadf55cd7f9cda1a1030026ddd42e905b77adc36e4f2d3c", - "8502091901001a0167aa07581dcba44f7f04f2de44f42d84c374a0e149136f25b01852545961d55f7f7a", - "8503091901001a0167aa07581d8cde6d0e2ec43f3b2dcb644a2209e8c9e34af5c4747984a5e873c9cf5f", - "8504091901001a0167aa07581d965e25ee29039fdf8ca74f1c769fc07eb7ebaec46e0695aea6cbd60b3e", - "8505091901001a0167aa07581dc4bbff1b9ffe8a9e7240129377b9d3711ed38d412fbb4442256f1e6f59", - "8506091901001a0167aa07581d5e0fc57fed451fb0a0101fb76b1fb1e1b88cfdfdaa946294a47de8fff1", - "8507091901001a0167aa07581d73f021c0e6f65b05c0a494e50791270a0050a73ae69b6725505a2ec8a5", - "8508091901001a0167aa07581d791457c9876dd34aadd192a53aa0dc66b556c0c215c7ceb8248b717c22", - "8509091901001a0167aa07581d951e65305b56a3706e3e86eb01c803bbf915d80edcd64d4d0000000000", - "850a091901001a0167aa07581d330f0f33a05eead4f331df229871bee733b50de71afd2e5a79f196de09", - "850b091901001a0167aa07581d3b205ce5e52d8c24a52cffa34c564fa1af3fdffcd349dc4258ee4ee828", - "850c091901001a0167aa07581ddd7bf725ea6c16d531b5f03254783803048ca08b87148daacd1cd7a006", - "850d091901001a0167aa07581d760be7ad1c6187902bbc04f539b9ee5eb8ea6833222edea36031306c01", - "850e091901001a0167aa07581d5bf4031217d2c3254b088fa7553778b5003632f46e21db129416f65b55", - "850f091901001a0167aa07581d73f021c0e6f65b05c0a494e50791270a0050a73ae69b6725505a2ec8a5", - "8510091901001a0167aa07581db8546ebfe2048541348910267331c643133f828afec9337c318f71b7df", - "8511091901001a0167aa07581d23dedeea74e3a0fb052befabefa13e2f80e4315c9dceed4c8630612e64", - "8512091901001a0167aa07581dd01a8daee769ce34b6b35d3ca0005302724abddae405bdb419c0a6b208", - "8513091901001a0167aa07581d3171c5dc365766eff25ae47c6f10e7de48cfb8474e050e5fe997a6dc24", - "8514091901001a0167aa07581de055c2433562184fa71b4be94f262e200f01c6f74c284b0dc6fae6673f" - ] + expected_parts = FOUNTAIN_ENCODER_CBOR_EXPECTED_PARTS + assert(parts == expected_parts) + + def test_file_fountain_encoder_cbor(self): + from ur.file_fountain_encoder import FileFountainEncoder + + message = make_message(256) + tmp_file_path = TMP_FILE + with open(tmp_file_path, "wb") as f: + f.write(message) + + encoder = FileFountainEncoder(tmp_file_path, 30) + parts = [] + for i in range(20): + parts.append(data_to_hex(encoder.next_part().cbor())) + + expected_parts = FOUNTAIN_ENCODER_CBOR_EXPECTED_PARTS assert(parts == expected_parts) def test_fountain_encoder_is_complete(self): @@ -427,6 +498,35 @@ def test_fountain_decoder(self): print(decoder.result) assert(False) + def test_file_fountain_decoder(self): + from ur.file_fountain_encoder import FileFountainEncoder + from ur.file_fountain_decoder import FileFountainDecoder + + message_seed = b"Wolf" + message_size = 32767 + max_fragment_len = 1000 + + message = make_message(message_size, message_seed) + with open(TMP_FILE, "wb") as f: + f.write(message) + encoder = FileFountainEncoder(TMP_FILE, max_fragment_len, 100) + decoder = FileFountainDecoder(".") + + # Emulating a do-while loop from the original reference code + while True: + part = encoder.next_part() + decoder.receive_part(part) + if decoder.is_complete(): + break + + if decoder.is_success(): + with open(decoder.result, "rb") as f: + text_result = f.read() + assert(text_result == message) + else: + print(decoder.result) + assert(False) + def test_fountain_cbor(self): part = Part(12, 8, 100, 0x12345678, bytes([1, 5, 3, 3 ,5])) cbor = part.cbor() @@ -476,28 +576,24 @@ def test_ur_encoder(self): for i in range(20): parts.append(encoder.next_part()) - expected_parts = [ - "ur:bytes/1-9/lpadascfadaxcywenbpljkhdcahkadaemejtswhhylkepmykhhtsytsnoyoyaxaedsuttydmmhhpktpmsrjtdkgslpgh", - "ur:bytes/2-9/lpaoascfadaxcywenbpljkhdcagwdpfnsboxgwlbaawzuefywkdplrsrjynbvygabwjldapfcsgmghhkhstlrdcxaefz", - "ur:bytes/3-9/lpaxascfadaxcywenbpljkhdcahelbknlkuejnbadmssfhfrdpsbiegecpasvssovlgeykssjykklronvsjksopdzmol", - "ur:bytes/4-9/lpaaascfadaxcywenbpljkhdcasotkhemthydawydtaxneurlkosgwcekonertkbrlwmplssjtammdplolsbrdzcrtas", - "ur:bytes/5-9/lpahascfadaxcywenbpljkhdcatbbdfmssrkzmcwnezelennjpfzbgmuktrhtejscktelgfpdlrkfyfwdajldejokbwf", - "ur:bytes/6-9/lpamascfadaxcywenbpljkhdcackjlhkhybssklbwefectpfnbbectrljectpavyrolkzczcpkmwidmwoxkilghdsowp", - "ur:bytes/7-9/lpatascfadaxcywenbpljkhdcavszmwnjkwtclrtvaynhpahrtoxmwvwatmedibkaegdosftvandiodagdhthtrlnnhy", - "ur:bytes/8-9/lpayascfadaxcywenbpljkhdcadmsponkkbbhgsoltjntegepmttmoonftnbuoiyrehfrtsabzsttorodklubbuyaetk", - "ur:bytes/9-9/lpasascfadaxcywenbpljkhdcajskecpmdckihdyhphfotjojtfmlnwmadspaxrkytbztpbauotbgtgtaeaevtgavtny", - "ur:bytes/10-9/lpbkascfadaxcywenbpljkhdcahkadaemejtswhhylkepmykhhtsytsnoyoyaxaedsuttydmmhhpktpmsrjtwdkiplzs", - "ur:bytes/11-9/lpbdascfadaxcywenbpljkhdcahelbknlkuejnbadmssfhfrdpsbiegecpasvssovlgeykssjykklronvsjkvetiiapk", - "ur:bytes/12-9/lpbnascfadaxcywenbpljkhdcarllaluzmdmgstospeyiefmwejlwtpedamktksrvlcygmzemovovllarodtmtbnptrs", - "ur:bytes/13-9/lpbtascfadaxcywenbpljkhdcamtkgtpknghchchyketwsvwgwfdhpgmgtylctotzopdrpayoschcmhplffziachrfgd", - "ur:bytes/14-9/lpbaascfadaxcywenbpljkhdcapazewnvonnvdnsbyleynwtnsjkjndeoldydkbkdslgjkbbkortbelomueekgvstegt", - "ur:bytes/15-9/lpbsascfadaxcywenbpljkhdcaynmhpddpzmversbdqdfyrehnqzlugmjzmnmtwmrouohtstgsbsahpawkditkckynwt", - "ur:bytes/16-9/lpbeascfadaxcywenbpljkhdcawygekobamwtlihsnpalnsghenskkiynthdzotsimtojetprsttmukirlrsbtamjtpd", - "ur:bytes/17-9/lpbyascfadaxcywenbpljkhdcamklgftaxykpewyrtqzhydntpnytyisincxmhtbceaykolduortotiaiaiafhiaoyce", - "ur:bytes/18-9/lpbgascfadaxcywenbpljkhdcahkadaemejtswhhylkepmykhhtsytsnoyoyaxaedsuttydmmhhpktpmsrjtntwkbkwy", - "ur:bytes/19-9/lpbwascfadaxcywenbpljkhdcadekicpaajootjzpsdrbalpeywllbdsnbinaerkurspbncxgslgftvtsrjtksplcpeo", - "ur:bytes/20-9/lpbbascfadaxcywenbpljkhdcayapmrleeleaxpasfrtrdkncffwjyjzgyetdmlewtkpktgllepfrltataztksmhkbot" - ] + expected_parts = UR_ENCODER_EXPECTED_PARTS + expected_parts = [s.upper() for s in expected_parts] + assert(parts == expected_parts) + + def test_file_ur_encoder(self): + from ur.file_ur_encoder import FileUREncoder + + ur = make_message_ur(256) + tmp_file_path = TMP_FILE + with open(tmp_file_path, "wb") as f: + f.write(ur.cbor) + ur.cbor = tmp_file_path + encoder = FileUREncoder(ur, 30) + parts = [] + for i in range(20): + parts.append(encoder.next_part()) + + expected_parts = UR_ENCODER_EXPECTED_PARTS expected_parts = [s.upper() for s in expected_parts] assert(parts == expected_parts) @@ -589,6 +685,48 @@ def test_multipart_ur(self): print('{}'.format(decoder.result)) assert(False) + def test_multipart_file_ur(self): + from ur.file_ur_encoder import FileUREncoder + from ur.file_ur_decoder import FileURDecoder + + ur = make_message_ur(32767) + old_cbor = ur.cbor + with open(TMP_FILE, "wb") as f: + f.write(ur.cbor) + ur.cbor = TMP_FILE + max_fragment_len = 1000 + first_seq_num = 100 + + # mock os.stat + import os + original_stat = os.stat + def real_stat(file_path): + l = [0] *11 + l[6] = original_stat(file_path).st_size + return l + with mock.patch("os.stat", side_effect=real_stat): + encoder = FileUREncoder(ur, max_fragment_len, first_seq_num) + decoder = FileURDecoder(".") + while True: + part = encoder.next_part() + decoder.receive_part(part) + if decoder.is_complete(): + break + + if decoder.is_success(): + with open(decoder.result.cbor, "rb") as f: + decoder.result.cbor = f.read() + # test FileUREncoder single UR + encoder = FileUREncoder(ur, 999999) + ur = URDecoder.decode(encoder.next_part()) + + assert(decoder.result.type == ur.type) + assert(decoder.result.cbor == ur.cbor) + assert(old_cbor == ur.cbor) + else: + print('{}'.format(decoder.result)) + assert(False) + def test_cbor_encode_decode_single(self): from ur.cbor_lite import CBOREncoder, CBORDecoder