zcbor.py: Add support for unordered maps in generated code

Add support for the --unordered-maps option to zcbor code
which makes use of the zcbor_unordered_maps_*() API in generated code.

Signed-off-by: Øyvind Rønningstad <[email protected]>

Author: oyvindronningstad

README.md

@@ -442,7 +442,7 @@ usage: zcbor code [-h] -c CDDL [--no-prelude] [-v]
                   -t ENTRY_TYPES [ENTRY_TYPES ...] [-d] [-e] [--time-header]
                   [--git-sha-header] [-b {32,64}]
                   [--include-prefix INCLUDE_PREFIX] [-s]
-                  [--file-header FILE_HEADER]
+                  [--file-header FILE_HEADER] [--unordered-maps]
 
 Parse a CDDL file and produce C code that validates and xcodes CBOR.
 The output from this script is a C file and a header file. The header file
@@ -543,6 +543,15 @@ options:
                         generated files, e.g. copyright. Can be a string or a
                         path to a file. If interpreted as a path to an
                         existing file, the file's contents will be used.
+  --unordered-maps      Add support in the generated code for decoding maps
+                        with unknown element order. When enabled, the
+                        generated code will use the zcbor_unordered_map_*()
+                        API to decode data whenever inside a map. zcbor
+                        detects when ZCBOR_MAP_SMART_SEARCH is needed and
+                        enables it in the generated cmake file. Enabling
+                        --unordered-maps places some restrictions on the level
+                        of ambiguity allowed between map keys in a map. Only
+                        affects decoding (--decode/-d).
 
 ```
 

include/zcbor_common.h

@@ -285,6 +285,7 @@ do { \
 #define ZCBOR_ERR_MAP_FLAGS_NOT_AVAILABLE 20
 #define ZCBOR_ERR_INVALID_VALUE_ENCODING 21 ///! When ZCBOR_CANONICAL is defined, and the incoming data is not encoded with minimal length, or uses indefinite length array.
 #define ZCBOR_ERR_CONSTANT_STATE_MISSING 22
+#define ZCBOR_ERR_NO_FLAG_MEM 23
 #define ZCBOR_ERR_UNKNOWN 31
 
 /** The largest possible elem_count. */
@@ -339,11 +340,22 @@ void zcbor_new_state(zcbor_state_t *state_array, size_t n_states,
 
 /** Do boilerplate entry function procedure.
  *  Initialize states, call function, and check the result.
+ *  This sets `manually_process_elem` to true.
  */
 int zcbor_entry_function(const uint8_t *payload, size_t payload_len,
 	void *result, size_t *payload_len_out, zcbor_state_t *state, zcbor_decoder_t func,
 	size_t n_states, size_t elem_count);
 
+
+/** Do boilerplate entry function procedure.
+ *  Initialize states, call function, and check the result.
+ *  Allows for using one or more state structs for map elem_states.
+ *  This sets `manually_process_elem` to true.
+ */
+int zcbor_entry_function_with_elem_states(const uint8_t *payload, size_t payload_len,
+	void *result, size_t *payload_len_out, zcbor_state_t *state, zcbor_decoder_t func,
+	size_t n_states, size_t elem_count, size_t n_elem_states);
+
 #ifdef ZCBOR_STOP_ON_ERROR
 /** Check stored error and fail if present, but only if stop_on_error is true.
  *
@@ -522,10 +534,15 @@ static inline size_t zcbor_flags_to_bytes(size_t num_flags)
 	(ZCBOR_ROUND_UP(num_flags, sizeof(zcbor_state_t) * ZCBOR_BITS_PER_BYTE) \
 			/ (sizeof(zcbor_state_t) * ZCBOR_BITS_PER_BYTE))
 
+#define ZCBOR_BYTES_TO_STATES(num_bytes) \
+	(ZCBOR_ROUND_UP(num_bytes, sizeof(zcbor_state_t)) / (sizeof(zcbor_state_t)))
+
+#define ZCBOR_BYTE_STATES(n_bytes) ZCBOR_BYTES_TO_STATES(n_bytes)
 #define ZCBOR_FLAG_STATES(n_flags) ZCBOR_FLAGS_TO_STATES(n_flags)
 
 #else
-#define ZCBOR_FLAG_STATES(n_flags) 0
+#define ZCBOR_BYTE_STATES(n_bytes) (n_bytes * 0)
+#define ZCBOR_FLAG_STATES(n_flags) (n_flags * 0)
 #endif
 
 size_t strnlen(const char *, size_t);

src/zcbor_common.c

@@ -142,6 +142,9 @@ void zcbor_new_state(zcbor_state_t *state_array, size_t n_states,
 	state_array[0].decode_state.indefinite_length_array = false;
 #ifdef ZCBOR_MAP_SMART_SEARCH
 	state_array[0].decode_state.map_search_elem_state = flags;
+	if (flags == NULL) {
+		flags_bytes = 0;
+	}
 	state_array[0].decode_state.map_elem_count = 0;
 #else
 	state_array[0].decode_state.map_elems_processed = 0;
@@ -299,11 +302,33 @@ size_t zcbor_remaining_str_len(zcbor_state_t *state)
 }
 
 
-int zcbor_entry_function(const uint8_t *payload, size_t payload_len,
+int zcbor_entry_function_with_elem_states(const uint8_t *payload, size_t payload_len,
 	void *result, size_t *payload_len_out, zcbor_state_t *state, zcbor_decoder_t func,
-	size_t n_states, size_t elem_count)
+	size_t n_states, size_t elem_count, size_t n_elem_states)
 {
-	zcbor_new_state(state, n_states, payload, payload_len, elem_count, NULL, 0);
+	uint8_t *flags = NULL;
+	size_t n_elem_state_bytes = 0;
+
+#ifdef ZCBOR_MAP_SMART_SEARCH
+	if (n_elem_states > 0) {
+		if (n_states < (ZCBOR_FLAG_STATES(n_elem_states) + 3)) {
+			return ZCBOR_ERR_NO_FLAG_MEM;
+		}
+
+		size_t flag_state_index =
+			n_states - ZCBOR_FLAG_STATES(n_elem_states);
+		flags =	(uint8_t *)&state[flag_state_index];
+		n_states = flag_state_index;
+		n_elem_state_bytes = zcbor_flags_to_bytes(n_elem_states);
+	}
+#else
+	(void)n_elem_states;
+#endif
+
+	zcbor_new_state(state, n_states, payload, payload_len, elem_count, flags,
+			n_elem_state_bytes);
+
+	state->constant_state->manually_process_elem = true;
 
 	bool ret = func(state, result);
 
@@ -321,6 +346,13 @@ int zcbor_entry_function(const uint8_t *payload, size_t payload_len,
 	return ZCBOR_SUCCESS;
 }
 
+int zcbor_entry_function(const uint8_t *payload, size_t payload_len,
+	void *result, size_t *payload_len_out, zcbor_state_t *state, zcbor_decoder_t func,
+	size_t n_states, size_t elem_count)
+{
+	return zcbor_entry_function_with_elem_states(payload, payload_len, result, payload_len_out,
+		state, func, n_states, elem_count, 0);
+}
 
 /* Float16: */
 #define F16_SIGN_OFFS 15 /* Bit offset of the sign bit. */

src/zcbor_decode.c

@@ -959,6 +959,10 @@ bool zcbor_unordered_map_search(zcbor_decoder_t key_decoder, zcbor_state_t *stat
 			(void)old_flags;
 		}
 
+		if (!should_try_key(state)) {
+			zcbor_log("Skipping element at index %zu.\n", get_current_index(state, 0));
+		}
+
 		if (should_try_key(state) && try_key(state, key_result, key_decoder)) {
 			if (!ZCBOR_MANUALLY_PROCESS_ELEM(state)) {
 				ZCBOR_FAIL_IF(!zcbor_elem_processed(state));
@@ -1576,8 +1580,8 @@ bool zcbor_multi_decode(size_t min_decode,
 			*num_decode = i;
 			state->payload = payload_bak;
 			state->elem_count = elem_count_bak;
-			ZCBOR_ERR_IF(i < min_decode, ZCBOR_ERR_ITERATIONS);
 			zcbor_log("Found %zu elements.\r\n", i);
+			ZCBOR_ERR_IF(i < min_decode, ZCBOR_ERR_ITERATIONS);
 			return true;
 		}
 	}

src/zcbor_print.c

@@ -93,6 +93,7 @@ const char *zcbor_error_str(int error)
 		ZCBOR_ERR_CASE(ZCBOR_ERR_MAP_FLAGS_NOT_AVAILABLE)
 		ZCBOR_ERR_CASE(ZCBOR_ERR_INVALID_VALUE_ENCODING)
 		ZCBOR_ERR_CASE(ZCBOR_ERR_CONSTANT_STATE_MISSING)
+		ZCBOR_ERR_CASE(ZCBOR_ERR_NO_FLAG_MEM)
 	}
 	#undef ZCBOR_ERR_CASE
 

tests/cases/serial_recovery.cddl

@@ -4,13 +4,11 @@
 ; SPDX-License-Identifier: Apache-2.0
 ;
 
-Member = ("image" => int) /
-	("data" => bstr) /
-	("len" => int) /
-	("off" => int) /
-	("sha" => bstr) /
-	(tstr => any)
-
 Upload = {
-	3*8members: Member
+	? "image" => uint,
+	? "len" => uint,
+	  "off" => uint,
+	? "sha" => bstr,
+	? "data" => bstr,
+	? "upgrade" => bool,
 }

tests/cases/unordered_map.cddl

@@ -0,0 +1,57 @@
+;
+; Copyright (c) 2023 Nordic Semiconductor ASA
+;
+; SPDX-License-Identifier: Apache-2.0
+;
+
+UnorderedMap1 = {
+	1 => "one",
+	2 => "two",
+	int => bstr,
+	"foo" => "bar",
+	bazboz,
+	*bstr => intlist,
+}
+
+bazboz = ("baz" => "boz")
+intlist = [*int]
+
+UnorderedMap2 = {
+	+typeUnion,
+	"map" => {
+		?-1 => bstr,
+		?-2 => bstr,
+		?bool => int,
+	}
+}
+
+typeUnion = type1 / type2 / typeDefault
+type1 = 1 => tstr
+type2 = 2 => tstr
+typeDefault = int => nil
+
+UnorderedMap3 = {
+	1*2000 uint => uint,
+	1*2000 nint => nint,
+}
+
+group1 = (
+	uint => tstr,
+	nint => bstr,
+)
+type3 = 3*3group1
+
+UnorderedMap4 = {
+	5*5group1,
+}
+
+UnorderedMap5 = {
+	1 => bool,
+	2 => {
+		-1 => int,
+		-2 => int,
+		-3 => UnorderedMap2,
+		*uint => bstr,
+	},
+
+}

tests/decode/test3_simple/CMakeLists.txt

@@ -35,6 +35,7 @@ set(py_command_serial_recovery
   -d
   ${bit_arg}
   --short-names
+  --unordered-maps
 
   # Testing the --include-prefix option
   --include-prefix serial

tests/decode/test3_simple/src/main.c

@@ -487,19 +487,15 @@ ZTEST(cbor_decode_test3, test_serial1)
 	zassert_equal(ZCBOR_SUCCESS, ret, "decoding failed: %d.", ret);
 	zassert_equal(sizeof(serial_rec_input1), decode_len, NULL);
 
-	zassert_equal(5, upload.members_count,
-		"expect 5 members");
-	zassert_equal(Member_data_c, upload.members[0].members
-		.Member_choice, "expect data 1st");
-	zassert_equal(Member_image_c, upload.members[1].members
-		.Member_choice, "expect image 2nd");
-	zassert_equal(Member_len_c, upload.members[2].members
-		.Member_choice, "was %d\r\n", upload.members[2].members
-		.Member_choice);
-	zassert_equal(Member_off_c, upload.members[3].members
-		.Member_choice, "expect off 4th");
-	zassert_equal(Member_sha_c, upload.members[4].members
-		.Member_choice, "expect sha 5th");
+	zassert_true(upload.data_present, NULL);
+	zassert_equal(0x129, upload.data.data.len, NULL);
+	zassert_true(upload.image_present, NULL);
+	zassert_equal(0, upload.image.image, NULL);
+	zassert_true(upload.len_present, NULL);
+	zassert_equal(0x3b2c, upload.len.len, NULL);
+	zassert_equal(0, upload.off, NULL);
+	zassert_true(upload.sha_present, NULL);
+	zassert_equal(0x20, upload.sha.sha.len, NULL);
 }
 
 ZTEST(cbor_decode_test3, test_serial2)
@@ -511,18 +507,15 @@ ZTEST(cbor_decode_test3, test_serial2)
 	zassert_equal(ZCBOR_SUCCESS, ret, "decoding failed: %d.", ret);
 	zassert_equal(sizeof(serial_rec_input2), decode_len, NULL);
 
-	zassert_equal(5, upload.members_count,
-		"expect 5 members");
-	zassert_equal(Member_data_c, upload.members[0].members
-		.Member_choice, "expect data 1st");
-	zassert_equal(Member_image_c, upload.members[1].members
-		.Member_choice, "expect image 2nd");
-	zassert_equal(Member_len_c, upload.members[2].members
-		.Member_choice, "expect len 3rd");
-	zassert_equal(Member_off_c, upload.members[3].members
-		.Member_choice, "expect off 4th");
-	zassert_equal(Member_sha_c, upload.members[4].members
-		.Member_choice, "expect sha 5th");
+	zassert_true(upload.data_present, NULL);
+	zassert_equal(0x129, upload.data.data.len, NULL);
+	zassert_true(upload.image_present, NULL);
+	zassert_equal(0, upload.image.image, NULL);
+	zassert_true(upload.len_present, NULL);
+	zassert_equal(0x2fe0, upload.len.len, NULL);
+	zassert_equal(0, upload.off, NULL);
+	zassert_true(upload.sha_present, NULL);
+	zassert_equal(0x20, upload.sha.sha.len, NULL);
 }
 
 ZTEST_SUITE(cbor_decode_test3, NULL, NULL, NULL, NULL, NULL);

tests/decode/test9_manifest14/CMakeLists.txt

@@ -8,6 +8,7 @@ cmake_minimum_required(VERSION 3.13.1)
 
 find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE})
 project(test9_manifest14)
+set(MAP_SEARCH_FLAGS ON)  # Because of --unordered maps
 include(../../cmake/test_template.cmake)
 
 if (NOT MANIFEST)
@@ -30,6 +31,7 @@ set(py_command
   SUIT_Common_Sequence
   -d
   ${bit_arg}
+  --unordered-maps
   )
 
 execute_process(