[sub-mac] redo security processing for every (re)transmission (openthread#13093)

Retransmissions of frames containing time-dependent header Information
Elements (IEs), such as CSL or Time Sync, require updates to these
IEs to reflect the exact time of sending. If the frame counter is not
incremented for these retransmissions, it leads to nonce reuse in
AES-CCM encryption, which is a security vulnerability.

This commit addresses this issue by ensuring that every transmission
attempt (initial or retry) uses a fresh frame counter:
- Deferred security processing from `SubMac::Send()` to
  `SubMac::BeginTransmit()`.
- Upon retransmission in `SubMac::HandleTransmitDone()`, the frame is
  restored to plaintext via `TxFrame::DecryptTransmitAesCcm()` and
  security flags are cleared.
- This allows time-dependent IEs to be updated and a new frame counter
  to be assigned for every attempt.

Added a Nexus test case `retransmission_security` to verify that both
CSL and standard MAC retransmissions use incrementing frame counters
and updated CSL phases.

Author: bukepo

examples/platforms/simulation/openthread-core-simulation-config.h

@@ -77,6 +77,9 @@
 #define OPENTHREAD_CONFIG_MAC_SOFTWARE_TX_TIMING_ENABLE 1
 #endif
 
+#ifndef OPENTHREAD_CONFIG_MAC_SOFTWARE_RETX_SECURITY_ENABLE
+#define OPENTHREAD_CONFIG_MAC_SOFTWARE_RETX_SECURITY_ENABLE 1
+#endif
 #endif // OPENTHREAD_RADIO
 
 #ifndef OPENTHREAD_CONFIG_PLATFORM_USEC_TIMER_ENABLE

examples/platforms/utils/mac_frame.cpp

@@ -404,6 +404,12 @@ void otMacFrameUpdateTimeIe(otRadioFrame *aFrame, uint64_t aRadioTime, otRadioCo
 
 otError otMacFrameProcessTxSfd(otRadioFrame *aFrame, uint64_t aRadioTime, otRadioContext *aRadioContext)
 {
+    otError error = OT_ERROR_NONE;
+
+    aFrame->mInfo.mTxInfo.mTimestamp = aRadioTime;
+
+    VerifyOrExit(!otMacFrameIsSecurityEnabled(aFrame) || !aFrame->mInfo.mTxInfo.mIsSecurityProcessed);
+
 #if OPENTHREAD_CONFIG_MAC_CSL_RECEIVER_ENABLE
     if (aRadioContext->mCslPresent) // CSL IE should be filled for every transmit attempt
     {
@@ -413,8 +419,10 @@ otError otMacFrameProcessTxSfd(otRadioFrame *aFrame, uint64_t aRadioTime, otRadi
 #if OPENTHREAD_CONFIG_TIME_SYNC_ENABLE
     otMacFrameUpdateTimeIe(aFrame, aRadioTime, aRadioContext);
 #endif
-    aFrame->mInfo.mTxInfo.mTimestamp = aRadioTime;
-    return otMacFrameProcessTransmitSecurity(aFrame, aRadioContext);
+    error = otMacFrameProcessTransmitSecurity(aFrame, aRadioContext);
+
+exit:
+    return error;
 }
 
 bool otMacFrameSrcAddrMatchCslReceiverPeer(const otRadioFrame *aFrame, const otRadioContext *aRadioContext)

src/core/config/mac.h

@@ -382,6 +382,15 @@
 #define OPENTHREAD_CONFIG_MAC_SOFTWARE_RX_ON_WHEN_IDLE_ENABLE 0
 #endif
 
+/**
+ * @def OPENTHREAD_CONFIG_MAC_SOFTWARE_RETX_SECURITY_ENABLE
+ *
+ * Define to 1 to enable software retransmission security logic.
+ */
+#ifndef OPENTHREAD_CONFIG_MAC_SOFTWARE_RETX_SECURITY_ENABLE
+#define OPENTHREAD_CONFIG_MAC_SOFTWARE_RETX_SECURITY_ENABLE 1
+#endif
+
 /**
  * @def OPENTHREAD_CONFIG_MAC_CSL_TRANSMITTER_ENABLE
  *

src/core/mac/mac_frame.cpp

@@ -41,7 +41,8 @@
 #include "common/log.hpp"
 #include "common/num_utils.hpp"
 #include "radio/trel_link.hpp"
-#if OPENTHREAD_FTD || OPENTHREAD_MTD || OPENTHREAD_CONFIG_MAC_SOFTWARE_TX_SECURITY_ENABLE
+#if OPENTHREAD_FTD || OPENTHREAD_MTD || OPENTHREAD_CONFIG_MAC_SOFTWARE_TX_SECURITY_ENABLE || \
+    (OPENTHREAD_CONFIG_MAC_HEADER_IE_SUPPORT && OPENTHREAD_CONFIG_MAC_SOFTWARE_RETX_SECURITY_ENABLE)
 #include "crypto/aes_ccm.hpp"
 #endif
 
@@ -1404,6 +1405,39 @@ void TxFrame::ProcessTransmitAesCcm(const ExtAddress &aExtAddress)
 #endif // OPENTHREAD_FTD || OPENTHREAD_MTD || OPENTHREAD_CONFIG_MAC_SOFTWARE_TX_SECURITY_ENABLE
 }
 
+#if OPENTHREAD_CONFIG_MAC_HEADER_IE_SUPPORT && OPENTHREAD_CONFIG_MAC_SOFTWARE_RETX_SECURITY_ENABLE
+void TxFrame::DecryptTransmitAesCcm(const ExtAddress &aExtAddress)
+{
+    uint32_t       frameCounter = 0;
+    uint8_t        securityLevel;
+    uint8_t        nonce[Crypto::AesCcm::kNonceSize];
+    uint8_t        tagLength;
+    Crypto::AesCcm aesCcm;
+
+    VerifyOrExit(GetSecurityEnabled() && IsSecurityProcessed());
+
+    SuccessOrExit(GetSecurityLevel(securityLevel));
+    SuccessOrExit(GetFrameCounter(frameCounter));
+
+    Crypto::AesCcm::GenerateNonce(aExtAddress, frameCounter, securityLevel, nonce);
+
+    aesCcm.SetKey(GetAesKey());
+    tagLength = GetFooterLength() - GetFcsSize();
+
+    aesCcm.Init(GetHeaderLength(), GetPayloadLength(), tagLength, nonce, sizeof(nonce));
+    aesCcm.Header(GetHeader(), GetHeaderLength());
+    aesCcm.Payload(GetPayload(), GetPayload(), GetPayloadLength(), Crypto::AesCcm::kDecrypt);
+    // Note: We skip aesCcm.Finalize() checking because we are only decrypting back to plaintext,
+    // and we know the ciphertext was generated correctly by us previously.
+
+    SetIsSecurityProcessed(false);
+    SetIsHeaderUpdated(false);
+
+exit:
+    return;
+}
+#endif // OPENTHREAD_CONFIG_MAC_HEADER_IE_SUPPORT && OPENTHREAD_CONFIG_MAC_SOFTWARE_RETX_SECURITY_ENABLE
+
 void TxFrame::GenerateImmAck(const RxFrame &aFrame, bool aIsFramePending)
 {
     uint16_t fcf = static_cast<uint16_t>(kTypeAck) | aFrame.GetVersion();

src/core/mac/mac_frame.hpp

@@ -626,6 +626,14 @@ class Frame : public otRadioFrame
 #endif // OPENTHREAD_CONFIG_TIME_SYNC_ENABLE
 
 #if OPENTHREAD_CONFIG_MAC_HEADER_IE_SUPPORT
+    /**
+     * Indicates whether the frame contains header IEs.
+     *
+     * @retval TRUE   The frame contains header IEs.
+     * @retval FALSE  The frame contains no header IEs.
+     */
+    bool HasHeaderIe(void) const { return FindHeaderIeIndex() != kInvalidIndex; }
+
     /**
      * Returns a pointer to the Header IE.
      *
@@ -1240,6 +1248,14 @@ class TxFrame : public Frame
      */
     void ProcessTransmitAesCcm(const ExtAddress &aExtAddress);
 
+    /**
+     * Decrypts the frame which was previously encrypted.
+     *
+     * @param[in]  aExtAddress  A reference to the extended address, which will be used to generate nonce
+     *                          for AES CCM computation.
+     */
+    void DecryptTransmitAesCcm(const ExtAddress &aExtAddress);
+
     /**
      * Indicates whether or not the frame has security processed.
      *

src/core/mac/sub_mac.cpp

@@ -63,6 +63,10 @@ SubMac::SubMac(Instance &aInstance)
     mCslParentAccuracy.Init();
 #endif
 
+#if OPENTHREAD_CONFIG_MAC_HEADER_IE_SUPPORT && !OPENTHREAD_CONFIG_MAC_SOFTWARE_RETX_SECURITY_ENABLE
+    // Assuming the platform must deal with the retransmission security correctly.
+    OT_ASSERT(mRadioCaps & OT_RADIO_CAPS_TRANSMIT_RETRIES);
+#endif
     Init();
 }
 
@@ -601,6 +605,15 @@ void SubMac::HandleTransmitDone(TxFrame &aFrame, RxFrame *aAckFrame, Error aErro
         mTransmitRetries++;
         aFrame.SetIsARetransmission(true);
 
+#if OPENTHREAD_CONFIG_MAC_HEADER_IE_SUPPORT && OPENTHREAD_CONFIG_MAC_SOFTWARE_RETX_SECURITY_ENABLE
+        if (aFrame.GetSecurityEnabled() && aFrame.IsSecurityProcessed() && aFrame.HasHeaderIe())
+        {
+            aFrame.DecryptTransmitAesCcm(GetExtAddress());
+        }
+
+        ProcessTransmitSecurity();
+#endif
+
 #if OPENTHREAD_CONFIG_MAC_ADD_DELAY_ON_NO_ACK_ERROR_BEFORE_RETRY
         if (aError == kErrorNoAck)
         {

tests/nexus/CMakeLists.txt

@@ -394,6 +394,7 @@ ot_nexus_test(1_4_PIC_TC_3 "cert;nexus")
 ot_nexus_test(1_4_PIC_TC_4 "cert;nexus")
 ot_nexus_test(1_4_CS_TC_3 "cert;nexus")
 ot_nexus_test(inform_previous_parent_on_reattach "cert;nexus")
+ot_nexus_test(retransmission_security "core;nexus")
 
 # Misc tests
 ot_nexus_test(anycast "core;nexus")

tests/nexus/openthread-core-nexus-config.h

@@ -74,6 +74,7 @@
 #define OPENTHREAD_CONFIG_DNS_CLIENT_ENABLE 1
 #define OPENTHREAD_CONFIG_DNS_CLIENT_BIND_UDP_TO_THREAD_NETIF 1
 #define OPENTHREAD_CONFIG_DNS_DSO_ENABLE 0
+#define OPENTHREAD_CONFIG_MAC_SOFTWARE_RETX_SECURITY_ENABLE 1
 #define OPENTHREAD_CONFIG_DNS_UPSTREAM_QUERY_ENABLE 1
 #define OPENTHREAD_CONFIG_DNSSD_DISCOVERY_PROXY_ENABLE 1
 #define OPENTHREAD_CONFIG_DNSSD_SERVER_ENABLE 1

tests/nexus/run_nexus_tests.sh

@@ -238,6 +238,7 @@ DEFAULT_TESTS=(
     "leader_reboot_multiple_link_request"
     "router_reboot_multiple_link_request"
     "pbbr_aloc"
+    "retransmission_security"
 )
 
 # Use provided arguments or the default test list

tests/nexus/test_retransmission_security.cpp

@@ -0,0 +1,135 @@
+/*
+ *  Copyright (c) 2026, The OpenThread Authors.
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions are met:
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. Neither the name of the copyright holder nor the
+ *     names of its contributors may be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ *  POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+
+#include "platform/nexus_core.hpp"
+#include "platform/nexus_node.hpp"
+
+namespace ot {
+namespace Nexus {
+
+/**
+ * Time to advance for a node to form a network and become leader, in milliseconds.
+ */
+static constexpr uint32_t kFormNetworkTime = 13 * 1000;
+
+/**
+ * Time to advance for a node to join as a SSED.
+ */
+static constexpr uint32_t kAttachAsSsedTime = 20 * 1000;
+
+/**
+ * CSL Period in milliseconds.
+ */
+static constexpr uint32_t kCslPeriodMs = 100;
+
+/**
+ * CSL Period in units of 10 symbols.
+ */
+static constexpr uint32_t kCslPeriod = kCslPeriodMs * 1000 / OT_US_PER_TEN_SYMBOLS;
+
+/**
+ * Time to advance for CSL synchronization to complete, in milliseconds.
+ */
+static constexpr uint32_t kCslSyncTime = 5 * 1000;
+
+/**
+ * Payload size for a standard ICMPv6 Echo Request.
+ */
+static constexpr uint16_t kEchoPayloadSize = 10;
+
+void TestRetransmissionSecurity(void)
+{
+    /**
+     * Retransmission Security Test
+     *
+     * Topology:
+     * - Leader (DUT)
+     * - SSED_1
+     *
+     * Purpose:
+     * Validate that retransmitted frames with CSL IE use an incremented frame counter
+     * and are correctly encrypted for each attempt.
+     */
+
+    Core nexus;
+
+    Node &leader = nexus.CreateNode();
+    Node &ssed1  = nexus.CreateNode();
+
+    leader.SetName("LEADER");
+    ssed1.SetName("SSED_1");
+
+    nexus.AdvanceTime(0);
+
+    SuccessOrQuit(Instance::SetGlobalLogLevel(kLogLevelNote));
+
+    Log("---------------------------------------------------------------------------------------");
+    Log("Step 1: Network Formation");
+
+    AllowLinkBetween(leader, ssed1);
+
+    leader.Form();
+    nexus.AdvanceTime(kFormNetworkTime);
+    VerifyOrQuit(leader.Get<Mle::Mle>().IsLeader());
+
+    ssed1.Join(leader, Node::kAsSed);
+    nexus.AdvanceTime(kAttachAsSsedTime);
+    VerifyOrQuit(ssed1.Get<Mle::Mle>().IsAttached());
+
+    ssed1.Get<Mac::Mac>().SetCslPeriod(kCslPeriod);
+    nexus.AdvanceTime(kCslSyncTime);
+    VerifyOrQuit(ssed1.Get<Mac::Mac>().IsCslEnabled());
+
+    Log("---------------------------------------------------------------------------------------");
+    Log("Step 2: Trigger Retransmissions (SSED to Leader)");
+
+    // Turn off leader radio so it misses SSED frames and SSED retries
+    SuccessOrQuit(otPlatRadioSleep(&leader.GetInstance()));
+
+    ssed1.SendEchoRequest(leader.Get<Mle::Mle>().GetMeshLocalEid(), 0x1234, kEchoPayloadSize);
+
+    // Advance time enough for multiple retries.
+    nexus.AdvanceTime(2000);
+
+    // Turn leader radio back on
+    SuccessOrQuit(otPlatRadioReceive(&leader.GetInstance(), leader.Get<Mac::Mac>().GetPanChannel()));
+
+    nexus.SaveTestInfo("test_retransmission_security.json");
+}
+
+} // namespace Nexus
+} // namespace ot
+
+int main(void)
+{
+    ot::Nexus::TestRetransmissionSecurity();
+    printf("All tests passed\n");
+    return 0;
+}