add the flag to use lightweight executor

Author: jeffnvidia

src/components/tl/ucp/allgather/allgather_knomial.c

@@ -422,6 +422,7 @@ ucc_status_t ucc_tl_ucp_allgather_knomial_init_r(
     ucc_tl_ucp_task_t *task;
     ucc_sbgp_t        *sbgp;
     ucc_status_t       status;
+    int                use_loopback = UCC_TL_UCP_TEAM_LIB(team)->cfg.allgather_use_loopback;
     task = ucc_tl_ucp_init_task(coll_args, team);
     status = ucc_mpool_init(&task->allgather_kn.etask_node_mpool, 0, sizeof(node_ucc_ee_executor_task_t),
                             0, UCC_CACHE_LINE_SIZE, 16, UINT_MAX, NULL,
@@ -440,7 +441,12 @@ ucc_status_t ucc_tl_ucp_allgather_knomial_init_r(
     }
     task->allgather_kn.etask_linked_list_head = NULL;
     task->allgather_kn.p.radix = radix;
-    task->super.flags         |= UCC_EE_EXECUTOR_PARAM_FIELD_TASK_TYPES ;
+    if (use_loopback){
+        task->super.flags         |= UCC_EE_EXECUTOR_PARAM_FIELD_TASK_TYPES;
+    }
+    else{
+        task->super.flags         |= UCC_COLL_TASK_FLAG_EXECUTOR;
+    }
     task->super.post           = ucc_tl_ucp_allgather_knomial_start;
     task->super.progress       = ucc_tl_ucp_allgather_knomial_progress;
     task->super.finalize       = ucc_tl_ucp_allgather_knomial_finalize;

src/components/tl/ucp/tl_ucp.c

@@ -140,6 +140,10 @@ ucc_config_field_t ucc_tl_ucp_lib_config_table[] = {
      ucc_offsetof(ucc_tl_ucp_lib_config_t, allgather_kn_radix),
      UCC_CONFIG_TYPE_UINT},
 
+     {"ALLGATHER_USE_LOOPBACK", "0", "If set to 1 performs network loopback for self copy, otherwise uses mc cuda copy",
+     ucc_offsetof(ucc_tl_ucp_lib_config_t, allgather_use_loopback),
+     UCC_CONFIG_TYPE_BOOL},
+
     {"BCAST_KN_RADIX", "4", "Radix of the recursive-knomial bcast algorithm",
      ucc_offsetof(ucc_tl_ucp_lib_config_t, bcast_kn_radix),
      UCC_CONFIG_TYPE_UINT},

src/components/tl/ucp/tl_ucp.h

@@ -56,6 +56,7 @@ typedef struct ucc_tl_ucp_lib_config {
     uint32_t                 reduce_scatter_kn_radix;
     uint32_t                 allgather_kn_radix;
     uint32_t                 bcast_kn_radix;
+    int                      allgather_use_loopback;
     ucc_mrange_uint_t        bcast_sag_kn_radix;
     uint32_t                 reduce_kn_radix;
     uint32_t                 gather_kn_radix;