tt

Author: wangraying

.buildkite/pipeline.yml

@@ -1,10 +1,11 @@
 steps:
   - label: "benchmark_master"
-    parallelism: 1
+    parallelism: 2
     command: bash .buildkite/scripts/benchmark_master.sh
     env:
       MASTER_ADDR: "10.158.66.134"
       MASTER_PORT: "29500"
+      CUDA_VISIBLE_DEVICES: "0,1,2,3"
     plugins:
       - docker#v3.8.0:
           image: "baguasys/bagua:master-pytorch-1.13.0-cuda11.6-cudnn8"
@@ -19,11 +20,12 @@ steps:
     agents:
       queue: "master"
   - label: "benchmark_worker"
-    parallelism: 1
+    parallelism: 2
     command: bash .buildkite/scripts/benchmark_worker.sh
     env:
       MASTER_ADDR: "10.158.66.134"
       MASTER_PORT: "29500"
+      CUDA_VISIBLE_DEVICES: "4,5,6,7"
     plugins:
       - docker#v3.8.0:
           image: "baguasys/bagua:master-pytorch-1.13.0-cuda11.6-cudnn8"
@@ -35,6 +37,8 @@ steps:
           ipc: host
           shm-size: 100gb
           always-pull: true
+    agents:
+      queue: "master"
   - label: "autotune_test"
     parallelism: 1
     command: bash .buildkite/scripts/benchmark.sh

.buildkite/scripts/benchmark_master.sh

@@ -129,7 +129,7 @@ function check_moe_log {
 
 MOE_SCRIPT="${WORKDIR}/examples/moe/mnist_main.py"
 logfile=$(mktemp /tmp/bagua_moe_gradient_allreduce.XXXXXX.log)
-NCCL_SOCKET_IFNAME=^docker,lo,veth CUDA_VISIBLE_DEVICES=0,1 python -m bagua.distributed.run \
+NCCL_SOCKET_IFNAME=^docker,lo,veth python -m bagua.distributed.run \
     --nnodes=2 \
     --nproc_per_node 2 \
     --rdzv_id=${BUILDKITE_BUILD_ID} \
@@ -146,7 +146,7 @@ check_moe_log ${logfile} 0.000071
 
 # 4. test moe checkpoint
 logfile=$(mktemp /tmp/bagua_moe_checkpoint.XXXXXX.log)
-CUDA_VISIBLE_DEVICES=0,1,2,3 python -m bagua.distributed.run \
+python -m bagua.distributed.run \
     --standalone \
     --nnodes=1 \
     --nproc_per_node 4 \

.buildkite/scripts/benchmark_worker.sh

@@ -5,8 +5,9 @@ printenv
 set -euox pipefail
 
 # 0. install bagua
-cp -a /upstream /workdir
-export WORKDIR=/workdir && cd $WORKDIR && bash .buildkite/scripts/install_bagua.sh || exit 1
+#cp -a /upstream /workdir
+#export WORKDIR=/workdir && cd $WORKDIR && bash .buildkite/scripts/install_bagua.sh || exit 1
+sleep 120
 
 # 1. test communication_primitives api
 echo "begin to test [communication_primitives]"
@@ -46,7 +47,7 @@ done
 # 3. test moe
 MOE_SCRIPT="${WORKDIR}/examples/moe/mnist_main.py"
 logfile=$(mktemp /tmp/bagua_moe_gradient_allreduce.XXXXXX.log)
-NCCL_SOCKET_IFNAME=^docker,lo,veth CUDA_VISIBLE_DEVICES=0,1 python -m bagua.distributed.run \
+NCCL_SOCKET_IFNAME=^docker,lo,veth python -m bagua.distributed.run \
     --nnodes=2 \
     --nproc_per_node 2 \
     --rdzv_id=${BUILDKITE_BUILD_ID} \

bagua/torch_api/communication.py

@@ -510,7 +510,7 @@ def init_process_group(store: Optional[torch.distributed.Store] = None, rank: in
         os.environ["LOCAL_WORLD_SIZE"] = str(local_world_size)
 
         _default_store = store
-        
+
     if _autotune_service_port is None:
         if get_rank() == 0:
             _autotune_service_port = _find_free_bagua_service_port(_default_store)

tests/internal/multi_process_v2.py

@@ -259,7 +259,7 @@ def run_test(self, test_name: str, parent_pipe) -> None:
                 f"Process {self.rank} skipping test {test_name} for following reason: {str(se)}"
             )
             sys.exit(TEST_SKIPS["generic"].exit_code)
-        except Exception as e:
+        except Exception:
             logger.error(
                 f"Caught exception: \n{traceback.format_exc()} exiting "
                 f"process {self.rank} with exit code: {MultiProcessTestCase.TEST_ERROR_EXIT_CODE}"

tests/torch_api/data_parallel/test_gradient_allreduce.py

@@ -111,7 +111,7 @@ def test_algorithm_hierarchical(self):
         torch.manual_seed(self.rank)
 
         self._init_bagua_distributed()
-        return run_model(hierarchical=true)
+        return run_model(hierarchical=True)
 
 
 if __name__ == "__main__":

tests/torch_api/test_gradient_allreduce.py

@@ -109,7 +109,7 @@ def test_algorithm_hierarchical(self):
         torch.manual_seed(self.rank)
 
         self._init_bagua_distributed()
-        return run_model(hierarchical=true)
+        return run_model(hierarchical=True)
 
 
 if __name__ == "__main__":