Squashed commit of the following:

commit c57bc71
Author: Fabio Battaglia <tabaglio@posteo.net>
Date:   Tue Sep 3 07:37:45 2024 +0200

    some more info print when dumping an IC

commit cb59ccd
Author: Fabio Battaglia <tabaglio@posteo.net>
Date:   Mon Sep 2 21:02:56 2024 +0200

    rewording of an error

commit 53f4672
Author: Fabio Battaglia <tabaglio@posteo.net>
Date:   Mon Sep 2 21:00:59 2024 +0200

    Reorganize error handling for cxfer

commit 1f7f955
Author: Fabio Battaglia <tabaglio@posteo.net>
Date:   Mon Sep 2 17:38:38 2024 +0200

    Bump to 0.4.0, remove old read code

commit 2d1ea9f
Author: Fabio Battaglia <tabaglio@posteo.net>
Date:   Mon Sep 2 17:19:17 2024 +0200

    Note the second pass to read the data

commit fbdc6d8
Author: Fabio Battaglia <tabaglio@posteo.net>
Date:   Mon Sep 2 16:34:35 2024 +0200

    current implementation of cxfer reading

Author: hkzlab

CHANGELOG.md

@@ -1,6 +1,11 @@
 # Changelog
 Changelog for the dpdumper utility
 
+## [0.4.0] - 2024-08-22
+### Changed
+- Depends on dupicolib 0.5.0
+- Reading ICs now uses a different protocol that cuts transfer time in roughly half
+
 ## [0.3.3] - 2024-08-22
 ### Changed
 - Depends on dpdumperlib 0.0.2

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dpdumper"
-version = "0.3.3"
+version = "0.4.0"
 description = "Tool to use the dupico as a dumping device"
 authors = [
    { name = "Fabio Battaglia", email = "hkzlabnet@gmail.com" }
@@ -18,7 +18,7 @@ classifiers = [
 requires-python = ">=3.12"
 dependencies = [
     "pyserial ~= 3.5",
-    "dupicolib >= 0.4.2",
+    "dupicolib >= 0.5.0",
     "dpdumperlib >= 0.0.2"
 ]
 

requirements.txt

@@ -1,3 +1,3 @@
 pyserial>=3.5
-dupicolib>=0.4.2
+dupicolib>=0.5.0
 dpdumperlib>=0.0.2

src/dpdumper/hl_board_utilities.py

@@ -1,11 +1,11 @@
 """This module contains high level utility code to perform operations on the board"""
 
-from typing import Generator, final, NamedTuple
+from typing import Callable, Generator, List, final, NamedTuple
+import logging
 
 import serial
-import math
-import logging
 
+import dupicolib.utils as DPLibUtils
 from dupicolib.hardware_board_commands import HardwareBoardCommands
 from dpdumperlib.ic.ic_definition import ICDefinition
 
@@ -44,6 +44,15 @@ def _read_pin_map_generator(cmd_class: type[HardwareBoardCommands], ic: ICDefini
             out_data_h: int = hi_pins_mapped | data_on_mapped | act_h_mapped | wr_l_mapped | address_mapped
             yield out_data_h
 
+def _build_update_callback(max_size: int) -> Callable[[int], None]:
+    def update_callback(cur_read: int) -> None:
+        if cur_read > max_size:
+            cur_read = max_size
+        _print_progressBar(cur_read, max_size)
+
+    return update_callback
+
+
 @final
 class DataElement(NamedTuple):
     data: int
@@ -60,50 +69,55 @@ class HLBoardUtilities:
     @classmethod
     def read_ic(cls, ser: serial.Serial, cmd_class: type[HardwareBoardCommands], ic: ICDefinition, check_hiz: bool = False) -> list[DataElement] | None:
         read_data: list[DataElement] = []
-        addr_combs: int = 1 << len(ic.address) # Calculate the number of addresses that this IC supports
-        wr_responses: list[int] = []
-        
-        _LOGGER.debug(f'read_ic command with definition {ic.name}, checking hi-z {check_hiz}. IC has {addr_combs} addresses and data width {len(ic.data)} bits.')
-
-        try:
-            hi_pins_mapped: int = cmd_class.map_value_to_pins(ic.adapter_hi_pins, 0xFFFFFFFFFFFFFFFF)
+        addr_combs: int = 1 << len(ic.address) # Calculate the number of addresses
+        data_width: int = -(len(ic.data) // -8)
+        dump_size: int = addr_combs * data_width
+        hi_pins: list[int] = list(set(ic.act_h_enable + ic.adapter_hi_pins))
 
-            _LOGGER.debug(f'This IC requires the following pin mask forced high: {hi_pins_mapped:0{16}X}')
+        data_normal: bytes | None = None
+        data_invert: bytes | None = None
 
-            cmd_class.write_pins(hi_pins_mapped, ser) # Start with these already enabled
-            cmd_class.set_power(True, ser)
+        upd_callback = _build_update_callback(dump_size)
 
-            tot_combs: int = addr_combs if not check_hiz else addr_combs * 2
+        _LOGGER.debug(f'read_ic command with definition {ic.name}, checking hi-z {check_hiz}. IC has {addr_combs} addresses and data width {len(ic.data)} bits.')
 
-            pin_map_gen = _read_pin_map_generator(cmd_class, ic, check_hiz)
-            
-            for i, pin_map in enumerate(pin_map_gen):
-                if i % 250 == 0:
-                    _print_progressBar(i, tot_combs)
+        print(f'IC has {addr_combs} addresses, for a total size of ~{-(dump_size//-1024)}KB.')
+        if check_hiz:
+            print('Read will be done in two passes to check for Hi-Z pins.')
 
-                wr_addr_response: int | None = cmd_class.write_pins(pin_map, ser)
+        try:
+            cmd_class.set_power(True, ser)
 
-                if wr_addr_response is None:
-                    return None # Something went wrong
-                else:
-                    wr_responses.append(wr_addr_response)
+            data_normal = cmd_class.cxfer_read(ic.address, ic.data, hi_pins, upd_callback, ser)
 
-            _print_progressBar(tot_combs, tot_combs)
+            if not data_normal:
+                raise IOError('Unable to read data from IC')
 
+            # If we need to check for Hi-Z, we need to forcefully set data pins to high and then redo the dump
             if check_hiz:
-                for pulled_low, pulled_up in grouped_iterator(wr_responses, 2):
-                    hiz_pins: int = pulled_low ^ pulled_up
-                    read_data.append(DataElement(data=cmd_class.map_pins_to_value(ic.data, pulled_low), z_mask=cmd_class.map_pins_to_value(ic.data, hiz_pins)))
-            else:
-                for pulled_low in wr_responses:
-                    read_data.append(DataElement(data=cmd_class.map_pins_to_value(ic.data, pulled_low)))
-        except Exception as exc:
-            _LOGGER.critical(exc)
+                print('Performing a second pass to detect Hi-Z pins!')
+                hi_pins = list(set(hi_pins + ic.data))
+                data_invert = cmd_class.cxfer_read(ic.address, ic.data, hi_pins, upd_callback, ser)
         finally:
-            print('') # Avoid writing again on the 'Testing block' string
+            ser.reset_input_buffer()
+            ser.reset_output_buffer()
             cmd_class.set_power(False, ser)
             cmd_class.write_pins(0, ser)
 
+        # Reconstruct the data elements
+        if data_normal and data_invert:
+            if len(data_normal) != len(data_invert):
+                raise IOError('Same IC read twice, bug got two dumps of different length!!!')
+
+            for dn, di in zip(DPLibUtils.iter_grouper(data_normal, data_width, 0), DPLibUtils.iter_grouper(data_invert, data_width, 0)):
+                dni = int.from_bytes(bytes(dn))
+                dii = int.from_bytes(bytes(di))
+                read_data.append(DataElement(data=dni, z_mask=dni^dii))
+        else:
+            for dn in DPLibUtils.iter_grouper(data_normal, data_width, 0):
+                dni = int.from_bytes(bytes(dn))
+                read_data.append(DataElement(data=dni))
+
         return read_data[:addr_combs]
 
     @staticmethod