cryptex and enc xtn hdr is not a valid combination

backport of the behavior in 3.x added in
  b40766c

Author: fippo

include/srtp.h

@@ -1759,7 +1759,8 @@ srtp_err_status_t srtp_get_stream_roc(srtp_t session,
  * @param enable whether to enable sending and receiving cryptex.
  *
  * @returns srtp_err_status_ok on success, or srtp_err_status_bad_param if the
- * stream or template cannot be found for the given SSRC.
+ * stream or template cannot be found for the given SSRC or if enabling cryptex
+ * would conflict with encrypted header extensions configured on the stream.
  */
 srtp_err_status_t srtp_set_stream_use_cryptex(srtp_t session,
                                               const srtp_ssrc_t *ssrc,

srtp/srtp.c

@@ -5155,6 +5155,9 @@ srtp_err_status_t srtp_set_stream_use_cryptex(srtp_t session,
         if (stream == NULL) {
             return srtp_err_status_bad_param;
         }
+        if (enable && stream->enc_xtn_hdr_count > 0) {
+            return srtp_err_status_bad_param;
+        }
         stream->use_cryptex = enable != 0;
         break;
     case ssrc_any_inbound:
@@ -5164,6 +5167,9 @@ srtp_err_status_t srtp_set_stream_use_cryptex(srtp_t session,
         if (session->stream_template == NULL) {
             return srtp_err_status_bad_param;
         }
+        if (enable && session->stream_template->enc_xtn_hdr_count > 0) {
+            return srtp_err_status_bad_param;
+        }
         session->stream_template->use_cryptex = enable != 0;
         data.template = session->stream_template;
         srtp_stream_list_for_each(session->stream_list,

test/srtp_driver.c

@@ -117,6 +117,8 @@ srtp_err_status_t srtp_test_cryptex_disable(void);
 
 srtp_err_status_t srtp_test_require_cryptex(void);
 
+srtp_err_status_t srtp_test_cryptex_and_enc_xtn_hdr_invalid(void);
+
 double srtp_bits_per_second(int msg_len_octets, const srtp_policy_t *policy);
 
 double srtp_rejections_per_second(int msg_len_octets,
@@ -694,6 +696,14 @@ int main(int argc, char *argv[])
             printf("failed\n");
             exit(1);
         }
+
+        printf("testing cryptex_and_enc_xtn_hdr_invalid()...");
+        if (srtp_test_cryptex_and_enc_xtn_hdr_invalid() == srtp_err_status_ok) {
+            printf("passed\n");
+        } else {
+            printf("failed\n");
+            exit(1);
+        }
     }
 
     if (do_stream_list) {
@@ -2758,6 +2768,42 @@ srtp_err_status_t srtp_test_require_cryptex(void)
     return srtp_err_status_ok;
 }
 
+srtp_err_status_t srtp_test_cryptex_and_enc_xtn_hdr_invalid(void)
+{
+    int headers[] = { 1 };
+
+    srtp_policy_t policy;
+    memset(&policy, 0, sizeof(policy));
+    srtp_crypto_policy_set_rtp_default(&policy.rtp);
+    srtp_crypto_policy_set_rtcp_default(&policy.rtcp);
+    policy.ssrc.type = ssrc_specific;
+    policy.ssrc.value = 0xcafebabe;
+    policy.key = test_key;
+    policy.window_size = 128;
+    policy.allow_repeat_tx = 0;
+    policy.enc_xtn_hdr = headers;
+    policy.enc_xtn_hdr_count = sizeof(headers) / sizeof(headers[0]);
+    policy.next = NULL;
+
+    srtp_t srtp_snd;
+    CHECK_OK(srtp_create(&srtp_snd, &policy));
+    CHECK_RETURN(srtp_set_stream_use_cryptex(srtp_snd, &policy.ssrc, 1),
+                 srtp_err_status_bad_param);
+    /* disabling is still allowed */
+    CHECK_OK(srtp_set_stream_use_cryptex(srtp_snd, &policy.ssrc, 0));
+    CHECK_OK(srtp_dealloc(srtp_snd));
+
+    policy.ssrc.type = ssrc_any_outbound;
+    srtp_t srtp_snd_wildcard;
+    CHECK_OK(srtp_create(&srtp_snd_wildcard, &policy));
+    CHECK_RETURN(
+        srtp_set_stream_use_cryptex(srtp_snd_wildcard, &policy.ssrc, 1),
+        srtp_err_status_bad_param);
+    CHECK_OK(srtp_dealloc(srtp_snd_wildcard));
+
+    return srtp_err_status_ok;
+}
+
 #ifdef GCM
 /*
  * srtp_validate_gcm() verifies the correctness of libsrtp by comparing