Add pjsua2 onAuthChallenge with sync fallback flag

Change async auth callback return type from void to pj_bool_t.
PJ_TRUE means app handles the challenge (async path), PJ_FALSE
lets the library fall through to synchronous reinit_req + send.
This allows pjsua2 to always register the callback while the
default no-op safely falls back to sync auth.

Add AuthChallenge class with respond()/abandon() methods and
OnAuthChallengeParam for the pjsua2 Account::onAuthChallenge()
virtual callback. Thread tdata and handled flag through pjsua
bridge layer.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: nanangizz

pjsip/include/pjsip/sip_auth.h

@@ -674,8 +674,14 @@ typedef struct pjsip_auth_clt_async_on_chal_param
  *                      to be passed to #pjsip_auth_clt_async_send_req().
  * @param param         The callback parameter containing the original request
  *                      and the response with the challenge.
+ *
+ * @return              PJ_TRUE if the application handles the challenge
+ *                      (will call pjsip_auth_clt_async_send_req() or
+ *                      pjsip_auth_clt_async_abandon() later).
+ *                      PJ_FALSE to let the library handle authentication
+ *                      via the synchronous path.
  */
-typedef void pjsip_auth_clt_async_on_challenge(
+typedef pj_bool_t pjsip_auth_clt_async_on_challenge(
                             pjsip_auth_clt_sess *sess,
                             void *token,
                             const pjsip_auth_clt_async_on_chal_param* param);

pjsip/include/pjsua-lib/pjsua.h

@@ -1281,6 +1281,21 @@ typedef struct pjsua_on_auth_challenge_param
      */
     const pjsip_rx_data        *rdata;
 
+    /**
+     * The original request that was challenged. Needed to build the
+     * authenticated retry. Only valid during the callback.
+     */
+    pjsip_tx_data              *tdata;
+
+    /**
+     * Output: set to PJ_TRUE if the application handles the challenge
+     * (will call pjsip_auth_clt_async_send_req() or
+     * pjsip_auth_clt_async_abandon() later).
+     * Default PJ_FALSE means the library handles authentication
+     * via the synchronous path.
+     */
+    pj_bool_t                   handled;
+
 } pjsua_on_auth_challenge_param;
 
 
@@ -2261,7 +2276,7 @@ typedef struct pjsua_callback
      * If this callback is not set, the library will handle authentication
      * automatically using the configured credentials (synchronous path).
      */
-    void (*on_auth_challenge)(const pjsua_on_auth_challenge_param *param);
+    void (*on_auth_challenge)(pjsua_on_auth_challenge_param *param);
 
 } pjsua_callback;
 

pjsip/include/pjsua-lib/pjsua_internal.h

@@ -1073,7 +1073,8 @@ void pjsua_acc_end_ip_change(pjsua_acc *acc);
 /*
  * Bridge callback: maps low-level auth challenge to pjsua on_auth_challenge.
  */
-void pjsua_auth_on_challenge(pjsip_auth_clt_sess *sess,
+pj_bool_t pjsua_auth_on_challenge(
+                             pjsip_auth_clt_sess *sess,
                              void *token,
                              const pjsip_auth_clt_async_on_chal_param *param);
 

pjsip/include/pjsua2/account.hpp

@@ -1881,6 +1881,58 @@ struct OnSendRequestParam
 };
 
 
+/**
+ * Represents a pending authentication challenge.
+ * Call respond() to resend with authentication, or abandon() to give up.
+ * If neither is called before the callback returns, the library handles
+ * authentication automatically using configured credentials (sync path).
+ * This object is only valid during the onAuthChallenge() callback.
+ */
+class AuthChallenge
+{
+public:
+    /**
+     * Respond to the authentication challenge by building and sending
+     * an authenticated request. Uses credentials currently configured
+     * on the account.
+     *
+     * @return          PJ_SUCCESS on success.
+     */
+    pj_status_t respond();
+
+    /**
+     * Abandon the authentication challenge. The pending request will
+     * not be resent.
+     *
+     * @return          PJ_SUCCESS on success.
+     */
+    pj_status_t abandon();
+
+private:
+    friend class Endpoint;
+
+    pjsua_on_auth_challenge_param *param;
+};
+
+/**
+ * Parameters for Account::onAuthChallenge() callback.
+ */
+struct OnAuthChallengeParam
+{
+    /** Account ID associated with the challenged request. */
+    pjsua_acc_id        accId;
+
+    /** Call ID, or PJSUA_INVALID_ID for non-call requests. */
+    pjsua_call_id       callId;
+
+    /** The 401/407 response containing the challenge. */
+    SipRxData           rdata;
+
+    /** The authentication challenge. Call respond() or abandon(). */
+    AuthChallenge       challenge;
+};
+
+
 /**
  * Parameters for presNotify() account method.
  */
@@ -2371,6 +2423,17 @@ class Account
     virtual void onMwiInfo(OnMwiInfoParam &prm)
     { PJ_UNUSED_ARG(prm); }
 
+    /**
+     * Called when a 401/407 challenge is received. Override to handle
+     * authentication asynchronously. Call prm.challenge.respond() to
+     * resend with authentication, or prm.challenge.abandon() to give up.
+     * If neither is called, the library handles it automatically.
+     *
+     * @param prm       Callback parameter.
+     */
+    virtual void onAuthChallenge(OnAuthChallengeParam &prm)
+    { PJ_UNUSED_ARG(prm); }
+
 private:
     friend class Endpoint;
     friend class Buddy;

pjsip/include/pjsua2/endpoint.hpp

@@ -2329,6 +2329,8 @@ class Endpoint
                            pjsua_acc_id acc_id);
     static void on_mwi_info(pjsua_acc_id acc_id,
                             pjsua_mwi_info *mwi_info);
+    static void on_auth_challenge(
+                            pjsua_on_auth_challenge_param *param);
     static void on_acc_find_for_incoming(const pjsip_rx_data *rdata,
                                          pjsua_acc_id* acc_id);
     static void on_buddy_state(pjsua_buddy_id buddy_id);

pjsip/src/pjsip/sip_auth_client.c

@@ -1965,6 +1965,13 @@ PJ_DEF(pj_status_t) pjsip_auth_clt_async_impl_on_challenge(
 
     cb_param           = *param;
     cb_param.user_data = sess->async_opt->user_data;
-    (*sess->async_opt->cb)(sess, token, &cb_param);
-    return PJ_SUCCESS;
+    if ((*sess->async_opt->cb)(sess, token, &cb_param)) {
+        return PJ_SUCCESS;
+    }
+
+    /* Callback chose not to handle — invalidate token, let caller
+     * fall through to synchronous path.
+     */
+    pj_bzero(token->signature, 4);
+    return PJ_EINVALIDOP;
 }

pjsip/src/pjsua-lib/pjsua_acc.c

@@ -36,7 +36,8 @@ static void schedule_reregistration(pjsua_acc *acc);
 static void keep_alive_timer_cb(pj_timer_heap_t *th, pj_timer_entry *te);
 
 /* Bridge: maps low-level auth challenge -> pjsua on_auth_challenge */
-void pjsua_auth_on_challenge(pjsip_auth_clt_sess *sess,
+pj_bool_t pjsua_auth_on_challenge(
+                             pjsip_auth_clt_sess *sess,
                              void *token,
                              const pjsip_auth_clt_async_on_chal_param *param)
 {
@@ -48,7 +49,8 @@ void pjsua_auth_on_challenge(pjsip_auth_clt_sess *sess,
 
     /* Determine call_id from rdata -> dialog -> mod_data */
     if (param->rdata) {
-        pjsip_dialog *dlg = pjsip_rdata_get_dlg(param->rdata);
+        pjsip_dialog *dlg = pjsip_rdata_get_dlg(
+                                        (pjsip_rx_data*)param->rdata);
         if (dlg) {
             pjsua_call *call =
                 (pjsua_call*)dlg->mod_data[pjsua_var.mod.id];
@@ -63,8 +65,11 @@ void pjsua_auth_on_challenge(pjsip_auth_clt_sess *sess,
     cb_param.auth_sess = sess;
     cb_param.token     = token;
     cb_param.rdata     = param->rdata;
+    cb_param.tdata     = param->tdata;
 
     (*pjsua_var.ua_cfg.cb.on_auth_challenge)(&cb_param);
+
+    return cb_param.handled;
 }
 
 /*

pjsip/src/pjsua2/endpoint.cpp

@@ -1164,6 +1164,51 @@ void Endpoint::on_mwi_info(pjsua_acc_id acc_id,
     acc->onMwiInfo(prm);
 }
 
+pj_status_t AuthChallenge::respond()
+{
+    pjsip_tx_data *new_tdata;
+    pj_status_t status;
+
+    status = pjsip_auth_clt_reinit_req(param->auth_sess,
+                                       (pjsip_rx_data*)param->rdata,
+                                       param->tdata, &new_tdata);
+    if (status == PJ_SUCCESS) {
+        status = pjsip_auth_clt_async_send_req(param->auth_sess,
+                                               param->token, new_tdata);
+    }
+    param->handled = PJ_TRUE;
+    return status;
+}
+
+pj_status_t AuthChallenge::abandon()
+{
+    pj_status_t status = pjsip_auth_clt_async_abandon(param->auth_sess,
+                                                      param->token);
+    param->handled = PJ_TRUE;
+    return status;
+}
+
+void Endpoint::on_auth_challenge(pjsua_on_auth_challenge_param *param)
+{
+    Account *acc = lookupAcc(param->acc_id, "on_auth_challenge()");
+    if (!acc) {
+        /* No account — leave handled=false, sync fallback */
+        return;
+    }
+
+    OnAuthChallengeParam prm;
+    prm.accId   = param->acc_id;
+    prm.callId  = param->call_id;
+    if (param->rdata)
+        prm.rdata.fromPj(*(pjsip_rx_data*)param->rdata);
+    prm.challenge.param = param;
+
+    acc->onAuthChallenge(prm);
+    /* If app called respond() or abandon(), param->handled is PJ_TRUE.
+     * If not (default no-op), param->handled stays PJ_FALSE → sync fallback.
+     */
+}
+
 void Endpoint::on_acc_find_for_incoming(const pjsip_rx_data *rdata,
                                         pjsua_acc_id* acc_id)
 {
@@ -2112,6 +2157,7 @@ void Endpoint::libInit(const EpConfig &prmEpConfig) PJSUA2_THROW(Error)
     ua_cfg.cb.on_rejected_incoming_call = &Endpoint::on_rejected_incoming_call;
     ua_cfg.cb.on_conf_op_completed      = &Endpoint::on_conf_op_completed;
     ua_cfg.cb.on_vid_conf_op_completed  = &Endpoint::on_vid_conf_op_completed;
+    ua_cfg.cb.on_auth_challenge         = &Endpoint::on_auth_challenge;
 
     /* Init! */
     PJSUA2_CHECK_EXPR( pjsua_init(&ua_cfg, &log_cfg, &med_cfg) );

pjsip/src/test/auth_async_test.c

@@ -169,9 +169,9 @@ static pj_status_t dummy_send_impl(pjsip_auth_clt_sess *sess,
  *****************************************************************************/
 
 /* Async challenge callback - used by the sync and deferred send tests. */
-static void on_auth_challenge(pjsip_auth_clt_sess *sess,
-                              void *token,
-                              const pjsip_auth_clt_async_on_chal_param *param)
+static pj_bool_t on_auth_challenge(pjsip_auth_clt_sess *sess,
+                                   void *token,
+                                   const pjsip_auth_clt_async_on_chal_param *param)
 {
     test_ctx_t    *ctx       = (test_ctx_t *)param->user_data;
     pjsip_tx_data *new_tdata = NULL;
@@ -189,7 +189,7 @@ static void on_auth_challenge(pjsip_auth_clt_sess *sess,
     ctx->reinit_status = status;
 
     if (status != PJ_SUCCESS || !new_tdata)
-        return;
+        return PJ_TRUE;
 
     if (ctx->sync) {
         /* Synchronous path: send from within the callback.
@@ -207,6 +207,7 @@ static void on_auth_challenge(pjsip_auth_clt_sess *sess,
          * async_auth_send_impl -> pjsip_regc_send.
          */
     }
+    return PJ_TRUE;
 }
 
 /* Registration completion callback. */
@@ -411,7 +412,7 @@ static struct {
     void                *token;
 } g_double_send;
 
-static void on_challenge_for_double_send(
+static pj_bool_t on_challenge_for_double_send(
                                 pjsip_auth_clt_sess *sess,
                                 void *token,
                                 const pjsip_auth_clt_async_on_chal_param *param)
@@ -422,7 +423,7 @@ static void on_challenge_for_double_send(
     status = pjsip_auth_clt_reinit_req(sess, param->rdata, param->tdata,
                                        &new_tdata);
     if (status != PJ_SUCCESS || !new_tdata)
-        return;
+        return PJ_TRUE;
 
     /* Remember the session and token before consuming them */
     g_double_send.sess  = sess;
@@ -431,6 +432,7 @@ static void on_challenge_for_double_send(
     /* First (and only legitimate) send - this clears token->signature */
     pjsip_auth_clt_async_send_req(sess, token, new_tdata);
     g_double_send.fired = PJ_TRUE;
+    return PJ_TRUE;
 }
 
 static int double_send_test(const pj_str_t *registrar_uri)
@@ -542,7 +544,7 @@ static struct {
     pj_bool_t fired;
 } g_abandoned;
 
-static void on_challenge_no_send(
+static pj_bool_t on_challenge_no_send(
                         pjsip_auth_clt_sess *sess,
                         void *token,
                         const pjsip_auth_clt_async_on_chal_param *param)
@@ -555,6 +557,7 @@ static void on_challenge_no_send(
      * pjsip_auth_clt_async_send_req.
      */
     g_abandoned.fired = PJ_TRUE;
+    return PJ_TRUE;
 }
 
 static int abandoned_token_test(const pj_str_t *registrar_uri)
@@ -636,7 +639,7 @@ static struct {
     void                *token;
 } g_abandon_api;
 
-static void on_challenge_do_abandon(
+static pj_bool_t on_challenge_do_abandon(
                         pjsip_auth_clt_sess *sess,
                         void *token,
                         const pjsip_auth_clt_async_on_chal_param *param)
@@ -650,6 +653,7 @@ static void on_challenge_do_abandon(
 
     /* Explicitly abandon: regc's abandon_impl will call on_reg_complete */
     pjsip_auth_clt_async_abandon(sess, token);
+    return PJ_TRUE;
 }
 
 static int abandon_api_test(const pj_str_t *registrar_uri)