infuse_iot: tdf: support index arrays

src/infuse_iot/generated/tdf_definitions.py

@@ -1298,6 +1298,42 @@ class lora_tx(TdfReadingBase):
             "payload": "{}",
         }
 
+    class idx_array_freq(TdfReadingBase):
+        """Sample frequency metadata for a TDF_DATA_FORMAT_IDX_ARRAY array"""
+
+        name = "IDX_ARRAY_FREQ"
+        _fields_ = [
+            ("tdf_id", ctypes.c_uint16),
+            ("frequency", ctypes.c_uint32),
+        ]
+        _pack_ = 1
+        _postfix_ = {
+            "tdf_id": "",
+            "frequency": "",
+        }
+        _display_fmt_ = {
+            "tdf_id": "{}",
+            "frequency": "{}",
+        }
+
+    class idx_array_period(TdfReadingBase):
+        """Sample frequency metadata for a TDF_DATA_FORMAT_IDX_ARRAY array"""
+
+        name = "IDX_ARRAY_PERIOD"
+        _fields_ = [
+            ("tdf_id", ctypes.c_uint16),
+            ("period", ctypes.c_uint32),
+        ]
+        _pack_ = 1
+        _postfix_ = {
+            "tdf_id": "",
+            "period": "",
+        }
+        _display_fmt_ = {
+            "tdf_id": "{}",
+            "period": "{}",
+        }
+
     class array_type(TdfReadingBase):
         """Example array type"""
 
@@ -1357,5 +1393,7 @@ class array_type(TdfReadingBase):
     43: readings.annotation,
     44: readings.lora_rx,
     45: readings.lora_tx,
+    46: readings.idx_array_freq,
+    47: readings.idx_array_period,
     100: readings.array_type,
 }

src/infuse_iot/tdf.py

@@ -3,6 +3,7 @@
 import copy
 import ctypes
 import enum
+import time
 from collections.abc import Generator
 
 from infuse_iot.generated import tdf_base, tdf_definitions
@@ -31,6 +32,8 @@ class flags(enum.IntEnum):
         TIMESTAMP_MASK = 0xC000
         TIME_ARRAY = 0x1000
         DIFF_ARRAY = 0x2000
+        IDX_ARRAY = 0x3000
+        ARRAY_MASK = 0x3000
         ID_MASK = 0x0FFF
 
     class DiffType(enum.IntEnum):
@@ -79,13 +82,15 @@ class Reading:
         def __init__(
             self,
             tdf_id: int,
-            time: None | float,
+            reading_time: None | float,
             period: None | float,
+            base_idx: None | int,
             data: list[tdf_base.TdfReadingBase],
         ):
             self.id = tdf_id
-            self.time = time
+            self.time = reading_time
             self.period = period
+            self.base_idx = base_idx
             self.data = data
 
     def __init__(self):
@@ -137,7 +142,7 @@ class _complete(ctypes.LittleEndianStructure):
         expanded = b"".join([bytes(b) for b in out])
         return ctypes.sizeof(raw), expanded
 
-    def decode(self, buffer: bytes) -> Generator[Reading, None, None]:
+    def decode(self, buffer: bytes, no_defs: bool = False) -> Generator[Reading, None, None]:
         buffer_time = None
 
         while len(buffer) > 3:
@@ -148,49 +153,73 @@ def decode(self, buffer: bytes) -> Generator[Reading, None, None]:
                 break
 
             tdf_id = header.id_flags & 0x0FFF
-            try:
-                id_type = tdf_definitions.id_type_mapping[tdf_id]
-            except KeyError:
+            if no_defs:
                 id_type = unknown_tdf_factory(tdf_id, header.len)
+            else:
+                try:
+                    id_type = tdf_definitions.id_type_mapping[tdf_id]
+                except KeyError:
+                    id_type = unknown_tdf_factory(tdf_id, header.len)
 
             if time_flags == self.flags.TIMESTAMP_NONE:
-                time = None
+                reading_time = None
             elif time_flags == self.flags.TIMESTAMP_ABSOLUTE:
                 t, buffer = self._buffer_pull(buffer, self.AbsoluteTime)
                 buffer_time = t.seconds * 65536 + t.subseconds
-                time = InfuseTime.unix_time_from_epoch(buffer_time)
+                reading_time = InfuseTime.unix_time_from_epoch(buffer_time)
             elif time_flags == self.flags.TIMESTAMP_RELATIVE:
                 t, buffer = self._buffer_pull(buffer, self.RelativeTime)
                 buffer_time += t.offset
-                time = InfuseTime.unix_time_from_epoch(buffer_time)
+                reading_time = InfuseTime.unix_time_from_epoch(buffer_time)
             elif time_flags == self.flags.TIMESTAMP_EXTENDED_RELATIVE:
                 t, buffer = self._buffer_pull(buffer, self.ExtendedRelativeTime)
                 buffer_time += t.offset
-                time = InfuseTime.unix_time_from_epoch(buffer_time)
+                reading_time = InfuseTime.unix_time_from_epoch(buffer_time)
             else:
                 raise RuntimeError("Unreachable time option")
 
             array_header = None
-            if header.id_flags & self.flags.DIFF_ARRAY:
+            base_idx = None
+            array_type = header.id_flags & self.flags.ARRAY_MASK
+            if array_type == self.flags.DIFF_ARRAY:
                 array_header, buffer = self._buffer_pull(buffer, self.ArrayHeader)
                 diff_type = array_header.num >> 6
                 diff_num = array_header.num & 0x3F
 
                 total_len, expanded = self._diff_expand(buffer, header.len, self.DiffType(diff_type), diff_num)
                 buffer = buffer[total_len:]
-                assert buffer_time is not None
-                time = InfuseTime.unix_time_from_epoch(buffer_time)
+                if buffer_time is None:
+                    t_now = int(time.time() * 65536)
+                    reading_time = InfuseTime.unix_time_from_epoch(t_now)
+                else:
+                    reading_time = InfuseTime.unix_time_from_epoch(buffer_time)
                 data = [
                     id_type.from_buffer_consume(expanded[x : x + header.len]) for x in range(0, total_len, header.len)
                 ]
-            elif header.id_flags & self.flags.TIME_ARRAY:
+            elif array_type == self.flags.TIME_ARRAY:
+                array_header, buffer = self._buffer_pull(buffer, self.ArrayHeader)
+                total_len = array_header.num * header.len
+                total_data = buffer[:total_len]
+                buffer = buffer[total_len:]
+
+                if buffer_time is None:
+                    t_now = int(time.time() * 65536)
+                    reading_time = InfuseTime.unix_time_from_epoch(t_now)
+                else:
+                    reading_time = InfuseTime.unix_time_from_epoch(buffer_time)
+                data = [
+                    id_type.from_buffer_consume(total_data[x : x + header.len]) for x in range(0, total_len, header.len)
+                ]
+            elif array_type == self.flags.IDX_ARRAY:
                 array_header, buffer = self._buffer_pull(buffer, self.ArrayHeader)
                 total_len = array_header.num * header.len
                 total_data = buffer[:total_len]
                 buffer = buffer[total_len:]
 
-                assert buffer_time is not None
-                time = InfuseTime.unix_time_from_epoch(buffer_time)
+                if time_flags != self.flags.TIMESTAMP_NONE:
+                    assert buffer_time is not None
+                    reading_time = InfuseTime.unix_time_from_epoch(buffer_time)
+                base_idx = array_header.period
                 data = [
                     id_type.from_buffer_consume(total_data[x : x + header.len]) for x in range(0, total_len, header.len)
                 ]
@@ -201,7 +230,29 @@ def decode(self, buffer: bytes) -> Generator[Reading, None, None]:
                 data = [id_type.from_buffer_consume(data_bytes)]
 
             period = None
-            if array_header is not None:
+            if array_header is not None and base_idx is None:
                 period = array_header.period / 65536
 
-            yield self.Reading(tdf_id, time, period, data)
+            yield self.Reading(tdf_id, reading_time, period, base_idx, data)
+
+
+if __name__ == "__main__":
+    import sys
+
+    if len(sys.argv) != 2:
+        sys.exit("Expected `python -m infuse_iot.tdf /path/to/tdf.bin`")
+
+    decoder = TDF()
+
+    with open(sys.argv[1], "rb") as f:
+        tdf_binary_blob = f.read(-1)
+
+        # Check data is in the form we expect (single block, 2 byte prefix)
+        if tdf_binary_blob[1] != 0x02:
+            sys.exit("Expected second byte to be 0x02 (INFUSE_TDF)")
+
+        try:
+            for tdf in decoder.decode(tdf_binary_blob[2:]):
+                print(f"TDF {tdf.id} @ t={tdf.time}: {tdf.data}")
+        except Exception:
+            pass

src/infuse_iot/tools/tdf_csv.py

@@ -54,13 +54,18 @@ def run(self):
                 # Construct reading strings
                 lines = []
                 reading_time = tdf.time
-                for reading in tdf.data:
+                for idx, reading in enumerate(tdf.data):
                     if self.args.unix:
                         time_func = _to_str
                     else:
                         time_func = InfuseTime.utc_time_string_log
 
-                    if reading_time is None:
+                    if tdf.base_idx is not None:
+                        if reading_time is None or idx > 0:
+                            time_str = f"{tdf.base_idx + idx}"
+                        else:
+                            time_str = time_func(reading_time)
+                    elif reading_time is None:
                         # Log with local time
                         time_str = time_func(time.time())
                     else:

src/infuse_iot/tools/tdf_list.py

@@ -59,7 +59,10 @@ def run(self) -> None:
                         offset = (len(tdf.data) - 1) * tdf.period
                         time_str = InfuseTime.utc_time_string(tdf.time + offset)
                 else:
-                    time_str = InfuseTime.utc_time_string(time.time())
+                    if tdf.base_idx is not None:
+                        time_str = f"IDX {tdf.base_idx}"
+                    else:
+                        time_str = InfuseTime.utc_time_string(time.time())
 
                 for field in t.iter_fields():
                     if isinstance(field.val, list):

tests/tdf/test_tdf.py

@@ -26,3 +26,38 @@ def test_tdf():
 
     # Number of TDFs on the example block should never change
     assert total_tdfs == 53
+
+
+def test_buffers():
+    # Valid buffers generated by `infuse-sdk/tests/subsys/tdf`
+    hex_strings = [
+        "6410040264003bdb31ccb6d508aa641004020a003bdb31ccb6d508aa",
+        "a00f103bdb31ccb6d508aad2e1be87e47b88ae",
+        "d0071d3bdb31ccb6d508aad2e1be87e47b88aeb19de6889efb2d975016f72831",
+        "6410040210803bdb31ccb6d508aa6410040210803bdb31ccb6d508aa",
+        "6400043bdb31cc64400440420f0000003bdb31cc",
+        "6430040100003bdb31cc64700440420f0000000101003bdb31cc",
+        "64400440420f0000003bdb31cc6400043bdb31cc",
+        "64700440420f0000000100003bdb31cc6430040101003bdb31cc",
+        "6410040696003bdb31ccb6d508aad2e1be87e47b88aeb19de6889efb2d97",
+        "6e400440420f0000003bdb31cc6fc004a086013bdb31cc",
+        "6e700440420f0000000100003bdb31cc6ff004a086010101003bdb31cc",
+        "32400640420f0000003bdb31ccb6d537c004ffffff3bdb31cc",
+        "32700640420f0000000100003bdb31ccb6d537f004ffffff0101003bdb31cc",
+        "14400440420f0000003bdb31cc1340043f421000ffff3bdb31cc",
+        "64400440420f0000003bdb31cc13400440420e0001003bdb31cc",
+        "64400440420f0000003bdb31cc648004ffff3bdb31cc648004ffff3bdb31cc",
+        "6400043bdb31cc6400043bdb31cc",
+        "6430040100003bdb31cc6430040101003bdb31cc",
+        "65400440420f0000003bdb31cc66800400003bdb31cc67800464003bdb31cc",
+    ]
+    decoder = TDF()
+
+    for hex_string in hex_strings:
+        total_tdfs = 0
+
+        # Iterate over each TDF in the buffer (not necessarily valid definition matches)
+        for tdf in decoder.decode(bytes.fromhex(hex_string), no_defs=True):
+            assert isinstance(tdf, TDF.Reading)
+            total_tdfs += 1
+        assert total_tdfs > 0