利用GYM Schedule进行SWE Agentic训练时出现模型迭代一轮后 退化全部生成“！！！！”

[key2long]

Checklist / 检查清单

• I have searched existing issues, and this is a new question or discussion topic. / 我已经搜索过现有的 issues，确认这是一个新的问题与讨论。

Question Description / 问题描述

利用Swfit中的GYM_Env进行多轮的SWE Agentic训练。当使用Deepspeed Zero3 + VLLM Server rollout进行GRPO的训练时（Qwen3.5-9B）模型在一个step后直接出现退化 输出全为 “！！！！！！“（应该是模型参数出现了NaN），但第一步的loss以及grad都是正常范围。但是使用Zero2训练时，一个step后是正常的。

相关zero3配置：{
"bf16": {
"enabled": true
},
"fp16": {
"enabled": false
},
"zero_optimization": {
"stage": 3,
"offload_optimizer": {
"device": "cpu",
"pin_memory": true
},
"overlap_comm": true,
"contiguous_gradients": true,
"stage3_max_live_parameters": 1e9,
"stage3_max_reuse_distance": 1e9,
"stage3_prefetch_bucket_size": 5e7,
"stage3_param_persistence_threshold": 1e5,
"reduce_bucket_size": 5e7,
"sub_group_size": 1e8,
"stage3_gather_16bit_weights_on_model_save": true
},
"gradient_accumulation_steps": "auto",
"gradient_clipping": 1.0,
"steps_per_print": 20,
"train_batch_size": "auto",
"train_micro_batch_size_per_gpu": "auto",
"wall_clock_breakdown": false
}

zero2配置：{
"bf16": {
"enabled": true
},
"fp16": {
"enabled": false
},
"zero_optimization": {
"stage": 2,
"offload_optimizer": {
"device": "cpu",
"pin_memory": true
},
"allgather_partitions": true,
"allgather_bucket_size": 20000000,
"overlap_comm": true,
"reduce_scatter": true,
"reduce_bucket_size": 20000000,
"contiguous_gradients": true
},
"gradient_accumulation_steps": "auto",
"gradient_clipping": 1.0,
"steps_per_print": 20,
"train_batch_size": "auto",
"train_micro_batch_size_per_gpu": "auto",
"wall_clock_breakdown": false
}

[key2long]

训练的配置：两台A800训练，另外一台机器的四张卡做Rollout。
# 训练脚本：
export PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF:-"max_split_size_mb:128,expandable_segments:True"}
export NCCL_BUFFSIZE=${NCCL_BUFFSIZE:-2097152}
export TRITON_PTXAS_EXTRA_ARGS=${TRITON_PTXAS_EXTRA_ARGS:-"-O0"}
export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-eth0}
export GLOO_SOCKET_IFNAME=${GLOO_SOCKET_IFNAME:-eth0}
export NCCL_TIMEOUT=${NCCL_TIMEOUT:-3600}
export TORCH_NCCL_BLOCKING_WAIT=${TORCH_NCCL_BLOCKING_WAIT:-1}
export NCCL_MAX_NRINGS=${NCCL_MAX_NRINGS:-4}
export NCCL_MIN_NCHANNELS=${NCCL_MIN_NCHANNELS:-2}
export NCCL_MAX_NCHANNELS=${NCCL_MAX_NCHANNELS:-4}
export NCCL_IB_DISABLE=${NCCL_IB_DISABLE:-1}
export NCCL_DEBUG=${NCCL_DEBUG:-INFO}
export NCCL_DEBUG_SUBSYS=${NCCL_DEBUG_SUBSYS:-NET,INIT}

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "${SCRIPT_DIR}/swe_traj_env.sh"
ENV_PLUGIN_PATH=${ENV_PLUGIN_PATH:-/mnt/tenant-home_speed/longxiao/workspace/mini-swe-agent/mini-swe-agent-main/src/minisweagent/environments/swift_train_env_apiv4.py}
SWIFT_SRC_DIR=${SWIFT_SRC_DIR:-/mnt/tenant-home_speed/longxiao/workspace/ms-swift-0428}
SWIFT_RLHF_CLI=${SWIFT_RLHF_CLI:-${SWIFT_SRC_DIR}/swift/cli/rlhf.py}
DEEPSPEED_CONFIG=${DEEPSPEED_CONFIG:-${SCRIPT_DIR}/deepspeed_zero2_para.json}
PYTHON_BIN=${PYTHON_BIN:-$(command -v python3 || command -v python || true)}

if [[ -z "${PYTHON_BIN}" ]]; then
echo "[train-dynamic-v4] python executable not found. Please set PYTHON_BIN=/path/to/python" >&2
exit 1
fi

if [[ ! -f "${SWIFT_RLHF_CLI}" ]]; then
echo "[train-dynamic-v4] SWIFT_RLHF_CLI not found: ${SWIFT_RLHF_CLI}" >&2
exit 1
fi
if [[ ! -f "${DEEPSPEED_CONFIG}" ]]; then
echo "[train-dynamic-v4] DEEPSPEED_CONFIG not found: ${DEEPSPEED_CONFIG}" >&2
exit 1
fi
TORCHRUN_BIN=${TORCHRUN_BIN:-$(command -v torchrun || true)}
if [[ -n "${TORCHRUN_BIN}" ]]; then
TRAIN_LAUNCHER=("${TORCHRUN_BIN}")
else
TRAIN_LAUNCHER=("${PYTHON_BIN}" -m torch.distributed.run)
fi

VLLM_HOST=${VLLM_HOST:-178.28.15.172}
VLLM_PORT=${VLLM_PORT:-8000}
SWE_ENV_IP=${SWE_ENV_IP:-127.0.0.1}
SWE_ENV_PORT=${SWE_ENV_PORT:-10026}
SWE_ENV_SERVER_URL=${SWE_ENV_SERVER_URL:-http://127.0.0.1:10026}

NNODES=${NNODES:-2}
NODE_RANK=${NODE_RANK:?please set NODE_RANK=0 or 1}
MASTER_ADDR=${MASTER_ADDR:-178.28.33.148}
MASTER_PORT=${MASTER_PORT:-29500}
NPROC_PER_NODE=${NPROC_PER_NODE:-8}

QWEN_MODEL_PATH=${QWEN_MODEL_PATH:-/mnt/tenant-home_speed/AIM/model/Qwen/Qwen3.5-9B}
SWE_CM_TOKENIZER_PATH=${SWE_CM_TOKENIZER_PATH:-${QWEN_MODEL_PATH}}
SWE_TEMPLATE=${SWE_TEMPLATE:-qwen3_5}
SWE_GRPO_MAX_MODEL_LEN=${SWE_GRPO_MAX_MODEL_LEN:-7768}
SWE_GRPO_MAX_COMPLETION_LENGTH=${SWE_GRPO_MAX_COMPLETION_LENGTH:-384}
SWE_GRPO_NUM_GENERATIONS=${SWE_GRPO_NUM_GENERATIONS:-4}
MODEL_NAME="$(basename "${QWEN_MODEL_PATH%/}")"

PER_DEVICE_TRAIN_BS=${PER_DEVICE_TRAIN_BS:-1}
PER_DEVICE_EVAL_BS=${PER_DEVICE_EVAL_BS:-1}
GRAD_ACC_STEPS=${GRAD_ACC_STEPS:-1}
STEPS_PER_GENERATION=${STEPS_PER_GENERATION:-1}
LEARNING_RATE=${LEARNING_RATE:-1e-7}
SWE_SAMPLING_TEMPERATURE=${SWE_SAMPLING_TEMPERATURE:-0.55}
SWE_SAMPLING_TOP_P=${SWE_SAMPLING_TOP_P:-0.92}
SWE_SAMPLING_TOP_K=${SWE_SAMPLING_TOP_K:-40}
SWE_GRPO_BETA=${SWE_GRPO_BETA:-0.06}
TRAIN_MODE=${TRAIN_MODE:-lora} # full|lm_head|lora

FREEZE_PARAMETERS_RATIO=${FREEZE_PARAMETERS_RATIO:-1}
TRAINABLE_PARAMETERS=${TRAINABLE_PARAMETERS:-lm_head}

LORA_RANK=${LORA_RANK:-8}
LORA_ALPHA=${LORA_ALPHA:-16}
LORA_DROPOUT=${LORA_DROPOUT:-0.05}
VLLM_ENABLE_LORA=${VLLM_ENABLE_LORA:-false}

ENABLE_FLATTENED_WEIGHT_SYNC=${ENABLE_FLATTENED_WEIGHT_SYNC:-false}
RUN_ID=${RUN_ID:-$(date +%Y%m%d-%H%M%S)}
OUTPUT_DIR=${OUTPUT_DIR:-${SCRIPT_DIR}/output/${MODEL_NAME}/v4-${RUN_ID}}
mkdir -p "${OUTPUT_DIR}"

--- rollout 服务消费 ---
SWE_CONTEXT_MANAGER=${SWE_CONTEXT_MANAGER:-swe_head_tail}
SWE_MULTI_TURN_SCHEDULER=${SWE_MULTI_TURN_SCHEDULER:-gym_scheduler_swe_dynamic}
SWE_DYNAMIC_HISTORY=${SWE_DYNAMIC_HISTORY:-1}
SWE_CM_MAX_INPUT_TOKENS=${SWE_CM_MAX_INPUT_TOKENS:-6000}
SWE_CM_HEAD_RATIO=${SWE_CM_HEAD_RATIO:-0.35}
ROLLOUT_MAX_TURNS=${ROLLOUT_MAX_TURNS:-15}
SWE_ROLLOUT_TIMEOUT=${SWE_ROLLOUT_TIMEOUT:-5400}
SWE_MAX_FORMAT_ERRORS=${SWE_MAX_FORMAT_ERRORS:-5}
SWE_REWARD_MODE=${SWE_REWARD_MODE:-v2}
SWE_ENABLE_SYMBOL_SPAM_FAST_FAIL=${SWE_ENABLE_SYMBOL_SPAM_FAST_FAIL:-1}
SWE_REWARD_FORMAT_ERROR=${SWE_REWARD_FORMAT_ERROR:--0.2}
SWE_REWARD_FORMAT_ERROR_LIMIT=${SWE_REWARD_FORMAT_ERROR_LIMIT:--1.5}

--- PRM (apiv4) ---
SWE_PRM_ENABLE=${SWE_PRM_ENABLE:-1}
SWE_PRM_EXTERNAL_ENABLE=${SWE_PRM_EXTERNAL_ENABLE:-0}
SWE_PRM_STEP_COEF=${SWE_PRM_STEP_COEF:-0.1}
SWE_PRM_CLIP_MIN=${SWE_PRM_CLIP_MIN:--1.5}
SWE_PRM_CLIP_MAX=${SWE_PRM_CLIP_MAX:-1.5}
SWE_PRM_API_BASE=${SWE_PRM_API_BASE:-http://10.55.42.83:31031/v1}

SWE_PRM_API_KEY=${SWE_PRM_API_KEY:-test-key}
SWE_PRM_MODEL=${SWE_PRM_MODEL:-minimax2.5}
SWE_PRM_TIMEOUT_S=${SWE_PRM_TIMEOUT_S:-30}
SWE_PRM_M=${SWE_PRM_M:-1}
SWE_PRM_MAX_HISTORY_STEPS=${SWE_PRM_MAX_HISTORY_STEPS:-8}
SWE_PRM_MAX_PROBLEM_LEN=${SWE_PRM_MAX_PROBLEM_LEN:-10000}
SWE_PRM_MAX_OUTPUT_LEN=${SWE_PRM_MAX_OUTPUT_LEN:-3000}
SWE_PRM_FALLBACK_TO_HEURISTIC=${SWE_PRM_FALLBACK_TO_HEURISTIC:-1}
QUICK_VERIFY_MODE=${QUICK_VERIFY_MODE:-0}
SWE_DEBUG_TOKEN_ALIGN=${SWE_DEBUG_TOKEN_ALIGN:-0}
SWE_DEBUG_STEPWISE_FULL=${SWE_DEBUG_STEPWISE_FULL:-1}
SWE_DEBUG_REWARD_ADV=${SWE_DEBUG_REWARD_ADV:-1}
SWIFT_DEBUG_REWARD_ADV=${SWIFT_DEBUG_REWARD_ADV:-1}
SWIFT_DEBUG_LOCAL_GENERATE_AFTER_STEP=${SWIFT_DEBUG_LOCAL_GENERATE_AFTER_STEP:-1}
SWIFT_DEBUG_LOCAL_GENERATE_STEPS=${SWIFT_DEBUG_LOCAL_GENERATE_STEPS:-0,1}

SWIFT_DEBUG_LOCAL_GENERATE_MAX_TOKENS=${SWIFT_DEBUG_LOCAL_GENERATE_MAX_TOKENS:-256}
SWIFT_DEBUG_LOCAL_GENERATE_SYSTEM_PROMPT=${SWIFT_DEBUG_LOCAL_GENERATE_SYSTEM_PROMPT:-}
SWIFT_DEBUG_LOCAL_GENERATE_PROMPT=${SWIFT_DEBUG_LOCAL_GENERATE_PROMPT:-"请用一句话回答：1+1等于几？"}
SWIFT_DEBUG_LOCAL_GENERATE_TEMPERATURE=${SWIFT_DEBUG_LOCAL_GENERATE_TEMPERATURE:-0}
SWIFT_DEBUG_LOCAL_GENERATE_TOP_P=${SWIFT_DEBUG_LOCAL_GENERATE_TOP_P:-1.0}
SWIFT_DEBUG_LOCAL_GENERATE_TOP_K=${SWIFT_DEBUG_LOCAL_GENERATE_TOP_K:-50}
SWIFT_DEBUG_LOCAL_GENERATE_REPETITION_PENALTY=${SWIFT_DEBUG_LOCAL_GENERATE_REPETITION_PENALTY:-1.0}
SWIFT_DEBUG_PARAM_STATS=${SWIFT_DEBUG_PARAM_STATS:-1}
SWIFT_DEBUG_PARAM_STATS_BEGIN_STEPS=${SWIFT_DEBUG_PARAM_STATS_BEGIN_STEPS:-0}
SWIFT_DEBUG_PARAM_STATS_END_STEPS=${SWIFT_DEBUG_PARAM_STATS_END_STEPS:-1}
SWIFT_DEBUG_PARAM_STATS_TARGETS=${SWIFT_DEBUG_PARAM_STATS_TARGETS:-lm_head.weight,model.embed_tokens.weight,embed_tokens.weight,model.norm.weight,norm.weight}
SWE_RESPONSE_PREFIX=${SWE_RESPONSE_PREFIX:-THOUGHT:}
SWE_FILTER_FORMAT_ERROR_SUPERVISION=${SWE_FILTER_FORMAT_ERROR_SUPERVISION:-1}
SWE_STRICT_FORMAT_ERROR_TERMINAL=${SWE_STRICT_FORMAT_ERROR_TERMINAL:-1}
SWE_FORCE_LEADING_MASK_ZERO=${SWE_FORCE_LEADING_MASK_ZERO:-1}
SWE_DYNAMIC_OUTPUT_MODE=${SWE_DYNAMIC_OUTPUT_MODE:-trajectory_single_sample}

STEPWISE_NO_EXTERNAL_PRM=${STEPWISE_NO_EXTERNAL_PRM:-0}
SWE_PRM_DISABLE=${SWE_PRM_DISABLE:-0}
if [[ "${STEPWISE_NO_EXTERNAL_PRM}" == "1" ]]; then
SWE_DYNAMIC_OUTPUT_MODE=openclaw_stepwise
SWE_PRM_EXTERNAL_ENABLE=0
SWE_DYNAMIC_HISTORY=1
echo "[train-dynamic-v4] STEPWISE_NO_EXTERNAL_PRM=1 -> SWE_DYNAMIC_OUTPUT_MODE=${SWE_DYNAMIC_OUTPUT_MODE} SWE_PRM_EXTERNAL_ENABLE=0 (heuristic PRM only if SWE_PRM_ENABLE=1)"
fi
if [[ "${SWE_PRM_DISABLE}" == "1" ]]; then
SWE_PRM_ENABLE=0
echo "[train-dynamic-v4] SWE_PRM_DISABLE=1 -> SWE_PRM_ENABLE=0 (outcome-only reward shaping off for PRM path)"
fi

if [[ "${SWE_PRM_EXTERNAL_ENABLE}" == "0" ]]; then
SWE_PRM_API_BASE=""
SWE_PRM_MODEL=""
SWE_PRM_FALLBACK_TO_HEURISTIC=1
fi

if [[ "${QUICK_VERIFY_MODE}" == "1" ]]; then
echo "[train-dynamic-v4] QUICK_VERIFY_MODE=1 enabled"
SWE_GRPO_NUM_GENERATIONS=2
SWE_GRPO_MAX_COMPLETION_LENGTH=192
ROLLOUT_MAX_TURNS=1
SWE_ROLLOUT_TIMEOUT=1200
STEPS_PER_GENERATION=1
PER_DEVICE_TRAIN_BS=1
PER_DEVICE_EVAL_BS=1
GRAD_ACC_STEPS=1
SWE_PRM_M=1
fi

EXTRA_TRAIN_ARGS=()
case "${TRAIN_MODE}" in
full)
EXTRA_TRAIN_ARGS+=(--tuner_type full)
;;
lm_head)
EXTRA_TRAIN_ARGS+=(--tuner_type full)
EXTRA_TRAIN_ARGS+=(--freeze_parameters_ratio "${FREEZE_PARAMETERS_RATIO}")
EXTRA_TRAIN_ARGS+=(--trainable_parameters "${TRAINABLE_PARAMETERS}")
;;
lora)
EXTRA_TRAIN_ARGS+=(--tuner_type lora)
EXTRA_TRAIN_ARGS+=(--lora_rank "${LORA_RANK}")
EXTRA_TRAIN_ARGS+=(--lora_alpha "${LORA_ALPHA}")
EXTRA_TRAIN_ARGS+=(--lora_dropout "${LORA_DROPOUT}")
;;
*)
echo "[train-dynamic-v4] Unsupported TRAIN_MODE=${TRAIN_MODE}. Use one of: full|lm_head|lora" >&2
exit 1
;;
esac

echo "[train-dynamic-v4] NODE_RANK=${NODE_RANK} NNODES=${NNODES} NPROC_PER_NODE=${NPROC_PER_NODE}"
echo "[train-dynamic-v4] MASTER=${MASTER_ADDR}:${MASTER_PORT}"
echo "[train-dynamic-v4] MODEL=${QWEN_MODEL_PATH}"
echo "[train-dynamic-v4] TEMPLATE=${SWE_TEMPLATE}"
echo "[train-dynamic-v4] OUTPUT_DIR=${OUTPUT_DIR}"
echo "[train-dynamic-v4] train_mode=${TRAIN_MODE} learning_rate=${LEARNING_RATE}"
if [[ "${TRAIN_MODE}" == "lora" ]]; then
echo "[train-dynamic-v4] lora_rank=${LORA_RANK} lora_alpha=${LORA_ALPHA} lora_dropout=${LORA_DROPOUT}"
echo "[train-dynamic-v4] vllm_enable_lora=${VLLM_ENABLE_LORA} enable_flattened_weight_sync=${ENABLE_FLATTENED_WEIGHT_SYNC}"
elif [[ "${TRAIN_MODE}" == "lm_head" ]]; then
echo "[train-dynamic-v4] freeze_parameters_ratio=${FREEZE_PARAMETERS_RATIO} trainable_parameters=${TRAINABLE_PARAMETERS}"
fi
echo "[train-dynamic-v4] scheduler=${SWE_MULTI_TURN_SCHEDULER} context_manager=${SWE_CONTEXT_MANAGER} dynamic_history=${SWE_DYNAMIC_HISTORY}"
echo "[train-dynamic-v4] cm_max_input_tokens=${SWE_CM_MAX_INPUT_TOKENS} cm_head_ratio=${SWE_CM_HEAD_RATIO}"
echo "[train-dynamic-v4] reward_mode=${SWE_REWARD_MODE} max_format_errors=${SWE_MAX_FORMAT_ERRORS}"
echo "[train-dynamic-v4] symbol_spam_fast_fail=${SWE_ENABLE_SYMBOL_SPAM_FAST_FAIL} reward_format_error=${SWE_REWARD_FORMAT_ERROR} reward_format_error_limit=${SWE_REWARD_FORMAT_ERROR_LIMIT}"
echo "[train-dynamic-v4] rollout_timeout_s=${SWE_ROLLOUT_TIMEOUT}"
echo "[train-dynamic-v4] prm_enable=${SWE_PRM_ENABLE} prm_external_enable=${SWE_PRM_EXTERNAL_ENABLE} prm_model=${SWE_PRM_MODEL:-none} prm_m=${SWE_PRM_M}"
echo "[train-dynamic-v4] sampling: temperature=${SWE_SAMPLING_TEMPERATURE} top_p=${SWE_SAMPLING_TOP_P} top_k=${SWE_SAMPLING_TOP_K} beta=${SWE_GRPO_BETA}"
echo "[train-dynamic-v4] add_non_thinking_prefix=false"
echo "[train-dynamic-v4] response_prefix=${SWE_RESPONSE_PREFIX}"
echo "[train-dynamic-v4] filter_format_error_supervision=${SWE_FILTER_FORMAT_ERROR_SUPERVISION}"
echo "[train-dynamic-v4] strict_format_error_terminal=${SWE_STRICT_FORMAT_ERROR_TERMINAL}"
echo "[train-dynamic-v4] force_leading_mask_zero=${SWE_FORCE_LEADING_MASK_ZERO}"
echo "[train-dynamic-v4] debug_reward_adv_env=${SWE_DEBUG_REWARD_ADV} debug_reward_adv_trainer=${SWIFT_DEBUG_REWARD_ADV}"
echo "[train-dynamic-v4] local_generate_probe=${SWIFT_DEBUG_LOCAL_GENERATE_AFTER_STEP} steps=${SWIFT_DEBUG_LOCAL_GENERATE_STEPS} max_tokens=${SWIFT_DEBUG_LOCAL_GENERATE_MAX_TOKENS}"
echo "[train-dynamic-v4] local_generate_sampling: temperature=${SWIFT_DEBUG_LOCAL_GENERATE_TEMPERATURE} top_p=${SWIFT_DEBUG_LOCAL_GENERATE_TOP_P} top_k=${SWIFT_DEBUG_LOCAL_GENERATE_TOP_K} repetition_penalty=${SWIFT_DEBUG_LOCAL_GENERATE_REPETITION_PENALTY}"
echo "[train-dynamic-v4] local_generate_system_prompt=${SWIFT_DEBUG_LOCAL_GENERATE_SYSTEM_PROMPT}"
echo "[train-dynamic-v4] local_generate_prompt=${SWIFT_DEBUG_LOCAL_GENERATE_PROMPT}"
echo "[train-dynamic-v4] param_stats_probe=${SWIFT_DEBUG_PARAM_STATS} begin_steps=${SWIFT_DEBUG_PARAM_STATS_BEGIN_STEPS} end_steps=${SWIFT_DEBUG_PARAM_STATS_END_STEPS} targets=${SWIFT_DEBUG_PARAM_STATS_TARGETS}"
echo "[train-dynamic-v4] quick_verify_mode=${QUICK_VERIFY_MODE} debug_token_align=${SWE_DEBUG_TOKEN_ALIGN} debug_stepwise_full=${SWE_DEBUG_STEPWISE_FULL} dynamic_output_mode=${SWE_DYNAMIC_OUTPUT_MODE}"
echo "[train-dynamic-v4] swift_src_dir=${SWIFT_SRC_DIR}"
echo "[train-dynamic-v4] swift_rlhf_cli=${SWIFT_RLHF_CLI}"
echo "[train-dynamic-v4] deepspeed_config=${DEEPSPEED_CONFIG}"
if [[ -n "${TORCHRUN_BIN}" ]]; then
echo "[train-dynamic-v4] torchrun=${TORCHRUN_BIN}"
else
echo "[train-dynamic-v4] torchrun not found; fallback to ${PYTHON_BIN} -m torch.distributed.run"
fi
echo "[train-dynamic-v4] external_plugin=${ENV_PLUGIN_PATH}"

CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}
PYTHONPATH="${SWIFT_SRC_DIR}${PYTHONPATH:+:${PYTHONPATH}}"
SWE_SAVE_TRAJ_DIR="${SWE_SAVE_TRAJ_DIR}"
SWE_ENV_IP=${SWE_ENV_IP}
SWE_ENV_PORT=${SWE_ENV_PORT}
SWE_ENV_SERVER_URL=${SWE_ENV_SERVER_URL}
SWE_DYNAMIC_HISTORY=${SWE_DYNAMIC_HISTORY}
SWE_CM_MAX_INPUT_TOKENS=${SWE_CM_MAX_INPUT_TOKENS}
SWE_CM_HEAD_RATIO=${SWE_CM_HEAD_RATIO}
SWE_CM_TOKENIZER_PATH=${SWE_CM_TOKENIZER_PATH}
ROLLOUT_MAX_TURNS=${ROLLOUT_MAX_TURNS}
SWE_ROLLOUT_TIMEOUT=${SWE_ROLLOUT_TIMEOUT}
SWE_MAX_FORMAT_ERRORS=${SWE_MAX_FORMAT_ERRORS}
SWE_REWARD_MODE=${SWE_REWARD_MODE}
SWE_ENABLE_SYMBOL_SPAM_FAST_FAIL=${SWE_ENABLE_SYMBOL_SPAM_FAST_FAIL}
SWE_REWARD_FORMAT_ERROR=${SWE_REWARD_FORMAT_ERROR}
SWE_REWARD_FORMAT_ERROR_LIMIT=${SWE_REWARD_FORMAT_ERROR_LIMIT}
SWE_PRM_ENABLE=${SWE_PRM_ENABLE}
SWE_PRM_EXTERNAL_ENABLE=${SWE_PRM_EXTERNAL_ENABLE}
SWE_PRM_STEP_COEF=${SWE_PRM_STEP_COEF}
SWE_PRM_CLIP_MIN=${SWE_PRM_CLIP_MIN}
SWE_PRM_CLIP_MAX=${SWE_PRM_CLIP_MAX}
SWE_PRM_API_BASE=${SWE_PRM_API_BASE}
SWE_PRM_API_KEY=${SWE_PRM_API_KEY}
SWE_PRM_MODEL=${SWE_PRM_MODEL}
SWE_PRM_TIMEOUT_S=${SWE_PRM_TIMEOUT_S}
SWE_PRM_M=${SWE_PRM_M}
SWE_PRM_MAX_HISTORY_STEPS=${SWE_PRM_MAX_HISTORY_STEPS}
SWE_PRM_MAX_PROBLEM_LEN=${SWE_PRM_MAX_PROBLEM_LEN}
SWE_PRM_MAX_OUTPUT_LEN=${SWE_PRM_MAX_OUTPUT_LEN}
SWE_PRM_FALLBACK_TO_HEURISTIC=${SWE_PRM_FALLBACK_TO_HEURISTIC}
SWE_DEBUG_TOKEN_ALIGN=${SWE_DEBUG_TOKEN_ALIGN}
SWE_DEBUG_STEPWISE_FULL=${SWE_DEBUG_STEPWISE_FULL}
SWE_DEBUG_REWARD_ADV=${SWE_DEBUG_REWARD_ADV}
SWIFT_DEBUG_REWARD_ADV=${SWIFT_DEBUG_REWARD_ADV}
SWIFT_DEBUG_LOCAL_GENERATE_AFTER_STEP=${SWIFT_DEBUG_LOCAL_GENERATE_AFTER_STEP}
SWIFT_DEBUG_LOCAL_GENERATE_STEPS=${SWIFT_DEBUG_LOCAL_GENERATE_STEPS}
SWIFT_DEBUG_LOCAL_GENERATE_MAX_TOKENS=${SWIFT_DEBUG_LOCAL_GENERATE_MAX_TOKENS}
SWIFT_DEBUG_LOCAL_GENERATE_TEMPERATURE=${SWIFT_DEBUG_LOCAL_GENERATE_TEMPERATURE}
SWIFT_DEBUG_LOCAL_GENERATE_TOP_P=${SWIFT_DEBUG_LOCAL_GENERATE_TOP_P}
SWIFT_DEBUG_LOCAL_GENERATE_TOP_K=${SWIFT_DEBUG_LOCAL_GENERATE_TOP_K}
SWIFT_DEBUG_LOCAL_GENERATE_REPETITION_PENALTY=${SWIFT_DEBUG_LOCAL_GENERATE_REPETITION_PENALTY}
SWIFT_DEBUG_LOCAL_GENERATE_SYSTEM_PROMPT="${SWIFT_DEBUG_LOCAL_GENERATE_SYSTEM_PROMPT}"
SWIFT_DEBUG_LOCAL_GENERATE_PROMPT="${SWIFT_DEBUG_LOCAL_GENERATE_PROMPT}"
SWIFT_DEBUG_PARAM_STATS=${SWIFT_DEBUG_PARAM_STATS}
SWIFT_DEBUG_PARAM_STATS_BEGIN_STEPS=${SWIFT_DEBUG_PARAM_STATS_BEGIN_STEPS}
SWIFT_DEBUG_PARAM_STATS_END_STEPS=${SWIFT_DEBUG_PARAM_STATS_END_STEPS}
SWIFT_DEBUG_PARAM_STATS_TARGETS=${SWIFT_DEBUG_PARAM_STATS_TARGETS}
SWE_FILTER_FORMAT_ERROR_SUPERVISION=${SWE_FILTER_FORMAT_ERROR_SUPERVISION}
SWE_STRICT_FORMAT_ERROR_TERMINAL=${SWE_STRICT_FORMAT_ERROR_TERMINAL}
SWE_FORCE_LEADING_MASK_ZERO=${SWE_FORCE_LEADING_MASK_ZERO}
SWE_DYNAMIC_OUTPUT_MODE=${SWE_DYNAMIC_OUTPUT_MODE}
NNODES=${NNODES}
NODE_RANK=${NODE_RANK}
MASTER_ADDR=${MASTER_ADDR}
MASTER_PORT=${MASTER_PORT}
NPROC_PER_NODE=${NPROC_PER_NODE}
DEEPSPEED_CONFIG="${DEEPSPEED_CONFIG}"
"${TRAIN_LAUNCHER[@]}"
--nnodes "${NNODES}"
--node_rank "${NODE_RANK}"
--nproc_per_node "${NPROC_PER_NODE}"
--master_addr "${MASTER_ADDR}"
--master_port "${MASTER_PORT}"
"${SWIFT_RLHF_CLI}"
--rlhf_type grpo
--max_steps 6
--model "${QWEN_MODEL_PATH}"
--template "${SWE_TEMPLATE}"
"${EXTRA_TRAIN_ARGS[@]}"
--use_vllm true
--vllm_mode server
--vllm_enable_lora ${VLLM_ENABLE_LORA}
--vllm_max_lora_rank ${LORA_RANK}
--enable_flattened_weight_sync ${ENABLE_FLATTENED_WEIGHT_SYNC}
--enable_thinking false
--add_non_thinking_prefix false
--response_prefix "${SWE_RESPONSE_PREFIX}"
--vllm_server_host ${VLLM_HOST}
--vllm_server_port ${VLLM_PORT}
--vllm_server_pass_dataset true
--external_plugins "${ENV_PLUGIN_PATH}"
--torch_dtype bfloat16
--dataset /mnt/tenant-home_speed/longxiao/workspace/mini-swe-agent/mini-swe-agent-main/data/swe_gym_subset_train_89_patch_only.jsonl
--load_from_cache_file true
--split_dataset_ratio 0
--max_model_len "${SWE_GRPO_MAX_MODEL_LEN}"
--max_completion_length "${SWE_GRPO_MAX_COMPLETION_LENGTH}"
--num_train_epochs 1
--per_device_train_batch_size ${PER_DEVICE_TRAIN_BS}
--per_device_eval_batch_size ${PER_DEVICE_EVAL_BS}
--learning_rate ${LEARNING_RATE}
--gradient_accumulation_steps ${GRAD_ACC_STEPS}
--steps_per_generation ${STEPS_PER_GENERATION}
--output_dir "${OUTPUT_DIR}"
--save_only_model true
--save_strategy steps
--save_steps 1
--save_total_limit 6
--logging_steps 1
--warmup_ratio 0.05
--dataloader_num_workers 2
--dataset_num_proc 2
--num_generations "${SWE_GRPO_NUM_GENERATIONS}"
--temperature ${SWE_SAMPLING_TEMPERATURE}
--top_p ${SWE_SAMPLING_TOP_P}
--top_k ${SWE_SAMPLING_TOP_K}
--log_completions true
--num_iterations 1
--report_to tensorboard
--beta ${SWE_GRPO_BETA}
--sequence_parallel_size 1
--deepspeed "${DEEPSPEED_CONFIG}"
--padding_free false
--attn_impl flash_attn
--gradient_checkpointing true
--loss_scale default
--lr_scheduler_type cosine
2>&1 | tee -a "${OUTPUT_DIR}/train_dynamic_v4_node${NODE_RANK}.log"

# Rollout脚本

SWIFT_SRC_DIR=${SWIFT_SRC_DIR:-/mnt/tenant-home_speed/longxiao/workspace/ms-swift-0428}
SWIFT_ROLLOUT_CLI=${SWIFT_ROLLOUT_CLI:-${SWIFT_SRC_DIR}/swift/cli/rollout.py}
ENV_PLUGIN_PATH=${ENV_PLUGIN_PATH:-/mnt/tenant-home_speed/longxiao/workspace/mini-swe-agent/mini-swe-agent-main/src/minisweagent/environments/swift_train_env_apiv4.py}
PYTHON_BIN=${PYTHON_BIN:-$(command -v python3 || command -v python || true)}

if [[ -z "${PYTHON_BIN}" ]]; then
echo "[rollout-dynamic-v4] python executable not found. Please set PYTHON_BIN=/path/to/python" >&2
exit 1
fi

if [[ ! -f "${SWIFT_ROLLOUT_CLI}" ]]; then
echo "[rollout-dynamic-v4] SWIFT_ROLLOUT_CLI not found: ${SWIFT_ROLLOUT_CLI}" >&2
exit 1
fi

if [[ ! -f "${ENV_PLUGIN_PATH}" ]]; then
echo "[rollout-dynamic-v4] ENV_PLUGIN_PATH not found: ${ENV_PLUGIN_PATH}" >&2
exit 1
fi

ROLLOUT_LOG_ENABLE=${ROLLOUT_LOG_ENABLE:-1}
ROLLOUT_LOG_DIR=${ROLLOUT_LOG_DIR:-"${SCRIPT_DIR}/rollout_service_logs"}
ROLLOUT_LOG_FILE=${ROLLOUT_LOG_FILE:-}

VLLM_HOST=${VLLM_HOST:-0.0.0.0}
VLLM_PORT=${VLLM_PORT:-8000}
QWEN_MODEL_PATH=${QWEN_MODEL_PATH:-/mnt/tenant-home_speed/AIM/model/Qwen/Qwen3.5-9B}
SWE_CM_TOKENIZER_PATH=${SWE_CM_TOKENIZER_PATH:-${QWEN_MODEL_PATH}}
SWE_TEMPLATE=${SWE_TEMPLATE:-qwen3_5}
SWE_RESPONSE_PREFIX=${SWE_RESPONSE_PREFIX:-THOUGHT:}

SWE_ENV_IP=${SWE_ENV_IP:-127.0.0.1}
SWE_ENV_PORT=${SWE_ENV_PORT:-10026}
SWE_ENV_SERVER_URL=${SWE_ENV_SERVER_URL:-http://127.0.0.1:10026}

SWE_GRPO_MAX_MODEL_LEN=${SWE_GRPO_MAX_MODEL_LEN:-7768}
VLLM_TP_SIZE=${VLLM_TP_SIZE:-4}
VLLM_GPU_UTIL=${VLLM_GPU_UTIL:-0.80}
ROLLOUT_MAX_TURNS=${ROLLOUT_MAX_TURNS:-15}
SWE_ROLLOUT_TIMEOUT=${SWE_ROLLOUT_TIMEOUT:-5400}

SWE_CONTEXT_MANAGER=${SWE_CONTEXT_MANAGER:-swe_head_tail}
SWE_MULTI_TURN_SCHEDULER=${SWE_MULTI_TURN_SCHEDULER:-gym_scheduler_swe_dynamic}
SWE_DYNAMIC_HISTORY=${SWE_DYNAMIC_HISTORY:-1}
SWE_CM_MAX_INPUT_TOKENS=${SWE_CM_MAX_INPUT_TOKENS:-6000}
SWE_CM_HEAD_RATIO=${SWE_CM_HEAD_RATIO:-0.35}
SWE_MAX_FORMAT_ERRORS=${SWE_MAX_FORMAT_ERRORS:-5}
SWE_REWARD_MODE=${SWE_REWARD_MODE:-v2}
SWE_ENABLE_SYMBOL_SPAM_FAST_FAIL=${SWE_ENABLE_SYMBOL_SPAM_FAST_FAIL:-1}
SWE_REWARD_FORMAT_ERROR=${SWE_REWARD_FORMAT_ERROR:--0.2}
SWE_REWARD_FORMAT_ERROR_LIMIT=${SWE_REWARD_FORMAT_ERROR_LIMIT:--1.5}

SWE_PRM_ENABLE=${SWE_PRM_ENABLE:-1}
SWE_PRM_EXTERNAL_ENABLE=${SWE_PRM_EXTERNAL_ENABLE:-0}
SWE_PRM_STEP_COEF=${SWE_PRM_STEP_COEF:-0.1}
SWE_PRM_CLIP_MIN=${SWE_PRM_CLIP_MIN:--1.5}
SWE_PRM_CLIP_MAX=${SWE_PRM_CLIP_MAX:-1.5}
SWE_PRM_API_BASE=${SWE_PRM_API_BASE:-http://10.55.42.83:31031/v1}

SWE_PRM_API_KEY=${SWE_PRM_API_KEY:-test-key}
SWE_PRM_MODEL=${SWE_PRM_MODEL:-minimax2.5}
SWE_PRM_TIMEOUT_S=${SWE_PRM_TIMEOUT_S:-30}
SWE_PRM_M=${SWE_PRM_M:-1}
SWE_PRM_MAX_HISTORY_STEPS=${SWE_PRM_MAX_HISTORY_STEPS:-8}
SWE_PRM_MAX_PROBLEM_LEN=${SWE_PRM_MAX_PROBLEM_LEN:-8000}
SWE_PRM_MAX_OUTPUT_LEN=${SWE_PRM_MAX_OUTPUT_LEN:-3000}
SWE_PRM_FALLBACK_TO_HEURISTIC=${SWE_PRM_FALLBACK_TO_HEURISTIC:-1}
QUICK_VERIFY_MODE=${QUICK_VERIFY_MODE:-0}
SWE_DEBUG_TOKEN_ALIGN=${SWE_DEBUG_TOKEN_ALIGN:-0}
SWE_DEBUG_STEPWISE_FULL=${SWE_DEBUG_STEPWISE_FULL:-1}
SWE_DEBUG_REWARD_ADV=${SWE_DEBUG_REWARD_ADV:-1}
SWE_FILTER_FORMAT_ERROR_SUPERVISION=${SWE_FILTER_FORMAT_ERROR_SUPERVISION:-1}
SWE_STRICT_FORMAT_ERROR_TERMINAL=${SWE_STRICT_FORMAT_ERROR_TERMINAL:-1}
SWE_FORCE_LEADING_MASK_ZERO=${SWE_FORCE_LEADING_MASK_ZERO:-1}
VLLM_ENABLE_LORA=${VLLM_ENABLE_LORA:-false}
VLLM_MAX_LORA_RANK=${VLLM_MAX_LORA_RANK:-8}

SWE_DYNAMIC_OUTPUT_MODE=${SWE_DYNAMIC_OUTPUT_MODE:-trajectory_single_sample}

STEPWISE_NO_EXTERNAL_PRM=1
STEPWISE_NO_EXTERNAL_PRM=${STEPWISE_NO_EXTERNAL_PRM:-0}
SWE_PRM_DISABLE=${SWE_PRM_DISABLE:-0}
if [[ "${STEPWISE_NO_EXTERNAL_PRM}" == "1" ]]; then
SWE_DYNAMIC_OUTPUT_MODE=openclaw_stepwise
SWE_PRM_EXTERNAL_ENABLE=0
SWE_DYNAMIC_HISTORY=1
echo "[rollout-dynamic-v4] STEPWISE_NO_EXTERNAL_PRM=1 -> SWE_DYNAMIC_OUTPUT_MODE=${SWE_DYNAMIC_OUTPUT_MODE} SWE_PRM_EXTERNAL_ENABLE=0"
fi
if [[ "${SWE_PRM_DISABLE}" == "1" ]]; then
SWE_PRM_ENABLE=0
echo "[rollout-dynamic-v4] SWE_PRM_DISABLE=1 -> SWE_PRM_ENABLE=0"
fi

if [[ "${SWE_PRM_EXTERNAL_ENABLE}" == "0" ]]; then
SWE_PRM_API_BASE=""
SWE_PRM_MODEL=""
SWE_PRM_FALLBACK_TO_HEURISTIC=1
fi

if [[ "${QUICK_VERIFY_MODE}" == "1" ]]; then
echo "[rollout-dynamic-v4] QUICK_VERIFY_MODE=1 enabled"
ROLLOUT_MAX_TURNS=1
SWE_ROLLOUT_TIMEOUT=1200
SWE_PRM_M=1
fi

export PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}
export NCCL_BUFFSIZE=${NCCL_BUFFSIZE:-2097152}
export NCCL_ASYNC_ERROR_HANDLING=${NCCL_ASYNC_ERROR_HANDLING:-1}
export TORCH_NCCL_BLOCKING_WAIT=${TORCH_NCCL_BLOCKING_WAIT:-1}
export NCCL_TIMEOUT=${NCCL_TIMEOUT:-1800}
export NCCL_DEBUG=${NCCL_DEBUG:-WARN}
export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-eth0}
export GLOO_SOCKET_IFNAME=${GLOO_SOCKET_IFNAME:-eth0}

if [[ "${ROLLOUT_LOG_ENABLE}" == "1" ]]; then
mkdir -p "${ROLLOUT_LOG_DIR}"
if [[ -z "${ROLLOUT_LOG_FILE}" ]]; then
ROLLOUT_LOG_FILE="${ROLLOUT_LOG_DIR}/swift_rollout_dynamic_v4_port${VLLM_PORT}$(date +%Y%m%d%H%M%S).log"
fi
touch "${ROLLOUT_LOG_FILE}"
exec > >(tee -a "${ROLLOUT_LOG_FILE}") 2>&1
fi

echo "[rollout-dynamic-v4] model=${QWEN_MODEL_PATH}"
echo "[rollout-dynamic-v4] template=${SWE_TEMPLATE}"
echo "[rollout-dynamic-v4] host=${VLLM_HOST} port=${VLLM_PORT} tp=${VLLM_TP_SIZE}"
echo "[rollout-dynamic-v4] scheduler=${SWE_MULTI_TURN_SCHEDULER} context_manager=${SWE_CONTEXT_MANAGER} dynamic_history=${SWE_DYNAMIC_HISTORY}"
echo "[rollout-dynamic-v4] cm_max_input_tokens=${SWE_CM_MAX_INPUT_TOKENS} cm_head_ratio=${SWE_CM_HEAD_RATIO}"
echo "[rollout-dynamic-v4] reward_mode=${SWE_REWARD_MODE} max_format_errors=${SWE_MAX_FORMAT_ERRORS}"
echo "[rollout-dynamic-v4] symbol_spam_fast_fail=${SWE_ENABLE_SYMBOL_SPAM_FAST_FAIL} reward_format_error=${SWE_REWARD_FORMAT_ERROR} reward_format_error_limit=${SWE_REWARD_FORMAT_ERROR_LIMIT}"
echo "[rollout-dynamic-v4] rollout_timeout_s=${SWE_ROLLOUT_TIMEOUT}"
echo "[rollout-dynamic-v4] prm_enable=${SWE_PRM_ENABLE} prm_external_enable=${SWE_PRM_EXTERNAL_ENABLE} prm_model=${SWE_PRM_MODEL:-none} prm_m=${SWE_PRM_M}"
echo "[rollout-dynamic-v4] add_non_thinking_prefix=false"
echo "[rollout-dynamic-v4] response_prefix=${SWE_RESPONSE_PREFIX}"
echo "[rollout-dynamic-v4] vllm_enable_lora=${VLLM_ENABLE_LORA} vllm_max_lora_rank=${VLLM_MAX_LORA_RANK}"
echo "[rollout-dynamic-v4] filter_format_error_supervision=${SWE_FILTER_FORMAT_ERROR_SUPERVISION}"
echo "[rollout-dynamic-v4] strict_format_error_terminal=${SWE_STRICT_FORMAT_ERROR_TERMINAL}"
echo "[rollout-dynamic-v4] force_leading_mask_zero=${SWE_FORCE_LEADING_MASK_ZERO}"
echo "[rollout-dynamic-v4] debug_reward_adv=${SWE_DEBUG_REWARD_ADV}"
echo "[rollout-dynamic-v4] quick_verify_mode=${QUICK_VERIFY_MODE} debug_token_align=${SWE_DEBUG_TOKEN_ALIGN} debug_stepwise_full=${SWE_DEBUG_STEPWISE_FULL} dynamic_output_mode=${SWE_DYNAMIC_OUTPUT_MODE}"
echo "[rollout-dynamic-v4] swift_src_dir=${SWIFT_SRC_DIR}"
echo "[rollout-dynamic-v4] swift_rollout_cli=${SWIFT_ROLLOUT_CLI}"
echo "[rollout-dynamic-v4] external_plugin=${ENV_PLUGIN_PATH}"

CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3}
PYTHONPATH="${SWIFT_SRC_DIR}${PYTHONPATH:+:${PYTHONPATH}}"
SWE_SAVE_TRAJ_DIR="${SWE_SAVE_TRAJ_DIR}"
SWE_ENV_IP=${SWE_ENV_IP}
SWE_ENV_PORT=${SWE_ENV_PORT}
SWE_ENV_SERVER_URL=${SWE_ENV_SERVER_URL}
SWE_DYNAMIC_HISTORY=${SWE_DYNAMIC_HISTORY}
SWE_CM_MAX_INPUT_TOKENS=${SWE_CM_MAX_INPUT_TOKENS}
SWE_CM_HEAD_RATIO=${SWE_CM_HEAD_RATIO}
SWE_CM_TOKENIZER_PATH=${SWE_CM_TOKENIZER_PATH}
ROLLOUT_MAX_TURNS=${ROLLOUT_MAX_TURNS}
SWE_ROLLOUT_TIMEOUT=${SWE_ROLLOUT_TIMEOUT}
SWE_MAX_FORMAT_ERRORS=${SWE_MAX_FORMAT_ERRORS}
SWE_REWARD_MODE=${SWE_REWARD_MODE}
SWE_ENABLE_SYMBOL_SPAM_FAST_FAIL=${SWE_ENABLE_SYMBOL_SPAM_FAST_FAIL}
SWE_REWARD_FORMAT_ERROR=${SWE_REWARD_FORMAT_ERROR}
SWE_REWARD_FORMAT_ERROR_LIMIT=${SWE_REWARD_FORMAT_ERROR_LIMIT}
SWE_PRM_ENABLE=${SWE_PRM_ENABLE}
SWE_PRM_EXTERNAL_ENABLE=${SWE_PRM_EXTERNAL_ENABLE}
SWE_PRM_STEP_COEF=${SWE_PRM_STEP_COEF}
SWE_PRM_CLIP_MIN=${SWE_PRM_CLIP_MIN}
SWE_PRM_CLIP_MAX=${SWE_PRM_CLIP_MAX}
SWE_PRM_API_BASE=${SWE_PRM_API_BASE}
SWE_PRM_API_KEY=${SWE_PRM_API_KEY}
SWE_PRM_MODEL=${SWE_PRM_MODEL}
SWE_PRM_TIMEOUT_S=${SWE_PRM_TIMEOUT_S}
SWE_PRM_M=${SWE_PRM_M}
SWE_PRM_MAX_HISTORY_STEPS=${SWE_PRM_MAX_HISTORY_STEPS}
SWE_PRM_MAX_PROBLEM_LEN=${SWE_PRM_MAX_PROBLEM_LEN}
SWE_PRM_MAX_OUTPUT_LEN=${SWE_PRM_MAX_OUTPUT_LEN}
SWE_PRM_FALLBACK_TO_HEURISTIC=${SWE_PRM_FALLBACK_TO_HEURISTIC}
SWE_DEBUG_TOKEN_ALIGN=${SWE_DEBUG_TOKEN_ALIGN}
SWE_DEBUG_STEPWISE_FULL=${SWE_DEBUG_STEPWISE_FULL}
SWE_DEBUG_REWARD_ADV=${SWE_DEBUG_REWARD_ADV}
SWE_FILTER_FORMAT_ERROR_SUPERVISION=${SWE_FILTER_FORMAT_ERROR_SUPERVISION}
SWE_STRICT_FORMAT_ERROR_TERMINAL=${SWE_STRICT_FORMAT_ERROR_TERMINAL}
SWE_FORCE_LEADING_MASK_ZERO=${SWE_FORCE_LEADING_MASK_ZERO}
SWE_DYNAMIC_OUTPUT_MODE=${SWE_DYNAMIC_OUTPUT_MODE}
"${PYTHON_BIN}" "${SWIFT_ROLLOUT_CLI}"
--host ${VLLM_HOST}
--port ${VLLM_PORT}
--model "${QWEN_MODEL_PATH}"
--template "${SWE_TEMPLATE}"
--enable_thinking false
--add_non_thinking_prefix false
--response_prefix "${SWE_RESPONSE_PREFIX}"
--dataset /mnt/tenant-home_speed/longxiao/workspace/mini-swe-agent/mini-swe-agent-main/data/swe_gym_subset_train_89_patch_only.jsonl
--infer_backend vllm
--vllm_tensor_parallel_size ${VLLM_TP_SIZE}
--vllm_enable_lora ${VLLM_ENABLE_LORA}
--vllm_max_lora_rank ${VLLM_MAX_LORA_RANK}
--use_gym_env true
--gym_env MiniSweEnv
--context_manager ${SWE_CONTEXT_MANAGER}
--vllm_gpu_memory_utilization ${VLLM_GPU_UTIL}
--vllm_max_model_len "${SWE_GRPO_MAX_MODEL_LEN}"
--multi_turn_scheduler ${SWE_MULTI_TURN_SCHEDULER}
--max_turns ${ROLLOUT_MAX_TURNS}
--external_plugins "${ENV_PLUGIN_PATH}"