zcbor_encode: Add new fragmented string encoding API

to match the redesigned decoding API.

Signed-off-by: Øyvind Rønningstad <[email protected]>

Author: oyvindronningstad

MIGRATION_GUIDE.md

@@ -13,6 +13,16 @@
 
   * A fix was made to the naming of bstr elements with a .size specifier, which might mean that these elements change name in your code when you regenerate.
 
+* The fragmented payload API has been completely redesigned to accomodate adding the encoding counterpart.
+  The docs have been updated and there's a new section in the README to explain the functionality.
+
+  * You must now define ZCBOR_FRAGMENTS to access the API
+  * `zcbor_*str_decode_fragment()` has been renamed to `zcbor_*str_fragments_start_decode()`
+  * After calling `zcbor_*str_fragments_start_decode()`, you must now retrieve the first fragment manually with `zcbor_str_fragment_decode()`, instead of via an argument.
+  * `zcbor_next_fragment()` and `zcbor_bstr_next_fragment()` have merged and is now called `zcbor_str_fragment_decode()`.
+    It does not take a `prev_fragment` argument, instead, this state is kept internally in the state struct.
+  * `zcbor_bstr_start_decode_fragment()` has been renamed to `zcbor_cbor_bstr_fragments_start_decode()` and does not return a fragment.
+    To retrieve fragments when decoding a CBOR-encoded bstr, use `zcbor_str_fragment_decode()`
 
 # zcbor v. 0.8.0
 

README.md

@@ -88,6 +88,31 @@ ZCBOR_STATE_D(decode_state, n, payload, payload_len, elem_count, n_flags);
 ZCBOR_STATE_E(encode_state, n, payload, payload_len, 0);
 ```
 
+Fragmented payloads
+-------------------
+
+zcbor can encode and decode payloads in sections, i.e. the payload can be split into separate buffers/arrays.
+This can be useful e.g. if you send or receive your payload in multiple packets.
+When the current payload section is done, call `zcbor_update_state()` to introduce the next section.
+Note that zcbor does not allow section boundaries to split a zcbor header/value pair.
+This means that the following elements cannot be split between sections:
+
+- Numbers and simple values (integers, floats, bools, undefined, nil)
+- Tags
+- Headers of lists, maps, tstrs, and bstrs
+
+If your payload is split in an unsupported way, you can get around it by making a small section out of the remaining bytes of one section spliced with the start of the next.
+Another option is to leave a little room at the start of each section buffer, and copy the remaining end of one section into the start of the next buffer.
+8 bytes should be enough for this.
+
+Lists and maps can span multiple sections, as long as the individual elements are not split as to break the above rule.
+
+String payloads can be split across multiple payload sections, if `ZCBOR_FRAGMENTS` is enabled, and the `*str_fragments_*()` APIs are used. Note that in the zcbor docs, the term "string fragment" is used for fragmented strings, while the term "payload section" is used for fragmented CBOR payloads, as passed to `zcbor_update_state()`. These do not always line up perfectly, particularly at the start and end of fragmented strings.
+
+CBOR-encoded bstrs can be nested, and there can also be a non-CBOR-encoded innermost string.
+The current innermost string (CBOR-encoded or otherwise) is called the "current string".
+`zcbor_update_state()` modifies all backups so that outer nested CBOR-encoded strings have updated information about the new section.
+
 Configuration
 -------------
 

include/zcbor_encode.h

@@ -231,6 +231,46 @@ bool zcbor_bstr_start_encode(zcbor_state_t *state);
  */
 bool zcbor_bstr_end_encode(zcbor_state_t *state, struct zcbor_string *result);
 
+
+#ifdef ZCBOR_FRAGMENTS
+
+/** Start encoding a fragmented string. I.e. a string spread over non-consecutive payload sections.
+ *
+ * After calling this, you can write a fragment with @ref zcbor_str_fragment_encode,
+ * then update the payload with @ref zcbor_update_state.
+ * Repeat until the string is fully decoded, then call @ref zcbor_bstr_fragments_end_encode.
+ */
+bool zcbor_bstr_fragments_start_encode(zcbor_state_t *state, size_t total_len);
+bool zcbor_tstr_fragments_start_encode(zcbor_state_t *state, size_t total_len);
+
+/** Start encoding a fragmented CBOR-encoded bytestring.
+ *
+ * I.e. a string spread over non-consecutive payload sections.
+ *
+ * This is an alternative to zcbor_*str_fragments_start_encode() to be used if the payload
+ * contains CBOR data that will be encoded directly with other zcbor_*() functions.
+ *
+ * A state backup is created to keep track of the element count and original payload_end.
+ * After calling this, you can encode elements using other zcbor functions,
+ * then update the payload with @ref zcbor_update_state.
+ * Repeat until the string is fully decoded, then call @ref zcbor_bstr_fragments_end_encode.
+ * When the current payload section contains the end of the string,
+ * payload_end is set to the end of the string, so there is no risk of encoding past the end.
+ */
+bool zcbor_cbor_bstr_fragments_start_encode(zcbor_state_t *state, size_t total_len);
+
+/** Retrieve a string fragment.
+ *
+ * Consume bytes from the payload until either the end of the payload or the end of the string.
+ * Do not use this function with @ref zcbor_cbor_bstr_fragments_start_encode.
+ */
+bool zcbor_str_fragment_encode(zcbor_state_t *state, struct zcbor_string *fragment, size_t *enc_len);
+
+/** Finish encoding a fragmented string. */
+bool zcbor_str_fragments_end_encode(zcbor_state_t *state);
+
+#endif /* ZCBOR_FRAGMENTS */
+
 #ifdef __cplusplus
 }
 #endif

src/zcbor_encode.c

@@ -48,6 +48,10 @@ static bool encode_header_byte(zcbor_state_t *state,
 
 	zcbor_assert_state(additional < 32, "Unsupported additional value: %d\r\n", additional);
 
+#ifdef ZCBOR_FRAGMENTS
+	ZCBOR_ERR_IF(state->inside_frag_str, ZCBOR_ERR_INSIDE_STRING);
+#endif
+
 	*(state->payload_mut) = (uint8_t)((major_type << 5) | (additional & 0x1F));
 	zcbor_trace(state, "value_encode");
 	state->payload_mut++;
@@ -234,6 +238,7 @@ bool zcbor_bstr_start_encode(zcbor_state_t *state)
 
 	/* Encode a dummy header */
 	if (!value_encode(state, ZCBOR_MAJOR_TYPE_BSTR, &max_len, sizeof(max_len))) {
+		zcbor_process_backup(state, ZCBOR_FLAG_CONSUME, 0xFFFFFFFF);
 		ZCBOR_FAIL();
 	}
 	return true;
@@ -259,7 +264,7 @@ bool zcbor_bstr_end_encode(zcbor_state_t *state, struct zcbor_string *result)
 	result->value = state->payload + zcbor_header_len(zcbor_remaining_str_len(state));
 	result->len = (size_t)payload - (size_t)result->value;
 
-	/* Reencode header of list now that we know the number of elements. */
+	/* Reencode header of list now that we know the length. */
 	if (!zcbor_bstr_encode(state, result)) {
 		ZCBOR_FAIL();
 	}
@@ -279,7 +284,7 @@ static bool str_encode(zcbor_state_t *state,
 	}
 	if (state->payload_mut != input->value) {
 		/* Use memmove since string might be encoded into the same space
-		 * because of bstrx_cbor_start_encode/bstrx_cbor_end_encode. */
+		 * because of zcbor_bstr_start_encode/zcbor_bstr_end_encode. */
 		memmove(state->payload_mut, input->value, input->len);
 	}
 	state->payload += input->len;
@@ -327,6 +332,106 @@ bool zcbor_tstr_put_term(zcbor_state_t *state, char const *str, size_t maxlen)
 }
 
 
+#ifdef ZCBOR_FRAGMENTS
+
+static bool start_encode_fragments(zcbor_state_t *state,
+	zcbor_major_type_t major_type, size_t len, bool cbor_bstr)
+{
+	ZCBOR_CHECK_PAYLOAD();
+
+	if (state->inside_cbor_bstr) {
+		if ((state->str_total_len_cbor - zcbor_current_string_offset(state) - zcbor_header_len(len)) < len) {
+			ZCBOR_ERR(ZCBOR_ERR_INNER_STRING_TOO_LARGE);
+		}
+	}
+
+	if (cbor_bstr) {
+		if (!zcbor_new_backup(state, 0)) {
+			ZCBOR_FAIL();
+		}
+	}
+
+	if (!value_encode(state, major_type, &len, sizeof(len))) {
+		if (cbor_bstr) {
+			zcbor_process_backup(state, ZCBOR_FLAG_CONSUME | ZCBOR_FLAG_RESTORE, 0xFFFFFFFF);
+		}
+		ZCBOR_FAIL();
+	}
+
+	ptrdiff_t new_offset = state->constant_state->curr_payload_section - state->payload;
+
+	if (cbor_bstr) {
+		state->frag_offset_cbor = new_offset;
+		state->str_total_len_cbor = len;
+		state->inside_cbor_bstr = true;
+	} else {
+		state->frag_offset = new_offset;
+		state->str_total_len = len;
+		state->inside_frag_str = true;
+	}
+
+	return true;
+}
+
+
+bool zcbor_bstr_fragments_start_encode(zcbor_state_t *state, size_t len)
+{
+	return start_encode_fragments(state, ZCBOR_MAJOR_TYPE_BSTR, len, false);
+}
+
+
+bool zcbor_tstr_fragments_start_encode(zcbor_state_t *state, size_t len)
+{
+	return start_encode_fragments(state, ZCBOR_MAJOR_TYPE_TSTR, len, false);
+}
+
+
+bool zcbor_cbor_bstr_fragments_start_encode(zcbor_state_t *state, size_t len)
+{
+	return start_encode_fragments(state, ZCBOR_MAJOR_TYPE_BSTR, len, true);
+}
+
+
+bool zcbor_str_fragment_encode(zcbor_state_t *state, struct zcbor_string *fragment, size_t *enc_len)
+{
+	ZCBOR_CHECK_PAYLOAD();
+
+	ZCBOR_ERR_IF(!state->inside_frag_str, ZCBOR_ERR_NOT_IN_FRAGMENT);
+
+	size_t len  = MIN(MIN((size_t)state->payload_end - (size_t)state->payload, fragment->len),
+				state->str_total_len - zcbor_current_string_offset(state));
+
+	memcpy(state->payload_mut, fragment->value, len);
+	state->payload += len;
+
+	if (enc_len != NULL) {
+		*enc_len = len;
+	}
+
+	return true;
+}
+
+
+bool zcbor_str_fragments_end_encode(zcbor_state_t *state)
+{
+	ZCBOR_ERR_IF(!state->inside_frag_str && !state->inside_cbor_bstr, ZCBOR_ERR_NOT_IN_FRAGMENT);
+	ZCBOR_ERR_IF(zcbor_current_string_remainder(state) != 0, ZCBOR_ERR_NOT_AT_END);
+
+	if (state->inside_frag_str) {
+		state->inside_frag_str = false;
+	} else {
+		if (!zcbor_process_backup(state, ZCBOR_FLAG_RESTORE | ZCBOR_FLAG_CONSUME | ZCBOR_FLAG_KEEP_PAYLOAD, 0xFFFFFFFF)) {
+			ZCBOR_FAIL();
+		}
+		state->elem_count++;
+	}
+
+	return true;
+}
+
+#endif /* ZCBOR_FRAGMENTS */
+
+
 static bool list_map_start_encode(zcbor_state_t *state, size_t max_num,
 		zcbor_major_type_t major_type)
 {
@@ -376,6 +481,14 @@ static bool list_map_end_encode(zcbor_state_t *state, size_t max_num,
 	size_t max_header_len = zcbor_header_len_ptr(&max_num, 4) - 1;
 	size_t header_len = zcbor_header_len_ptr(&list_count, 4) - 1;
 
+	if (max_num == list_count) {
+		if (!zcbor_process_backup(state, ZCBOR_FLAG_RESTORE | ZCBOR_FLAG_CONSUME | ZCBOR_FLAG_KEEP_PAYLOAD, 0xFFFFFFFF)) {
+			ZCBOR_FAIL();
+		}
+		state->elem_count++;
+		return true;
+	}
+
 	if (!zcbor_process_backup(state, ZCBOR_FLAG_RESTORE | ZCBOR_FLAG_CONSUME, 0xFFFFFFFF)) {
 		ZCBOR_FAIL();
 	}