Fix max_turns

examples/sglang_multiturn/config/gsm8k_multiturn_grpo_w_interaction.yaml

@@ -0,0 +1,23 @@
+hydra:
+  searchpath:
+    - file://verl/trainer/config
+
+defaults:
+  - ppo_trainer
+  - _self_
+
+data:
+  max_prompt_length: 1024
+  max_response_length: 1024
+  train_batch_size: 256
+  return_raw_chat: True
+
+actor_rollout_ref:
+  hybrid_engine: True
+  rollout:
+    name: sglang
+    multi_turn:
+      enable: True
+      max_user_turns: 5
+      format: qwen
+      # tool_config_path: "./config/tool_config/gsm8k_tool_config.yaml"

examples/sglang_multiturn/config/retool_multiturn_grpo.yaml

@@ -18,5 +18,5 @@ actor_rollout_ref:
     name: sglang_async
     multi_turn:
       enable: True
-      max_turns: 5
+      max_assistant_turns: 5
       tool_config_path: "./config/tool_config/sandbox_fusion_tool_config.yaml"

examples/sglang_multiturn/config/search_multiturn_grpo.yaml

@@ -19,5 +19,5 @@ actor_rollout_ref:
     name: sglang_async
     multi_turn:
       enable: True
-      max_turns: 2
+      max_assistant_turns: 2
       format: qwen

examples/sglang_multiturn/run_qwen2.5-3b_gsm8k_multiturn_w_interaction_mb.sh

@@ -27,7 +27,7 @@ HOME=/user/longxiang1
 
 python3 -m verl.trainer.main_ppo \
     --config-path="$CONFIG_PATH" \
-    --config-name='gsm8k_multiturn_grpo' \
+    --config-name='gsm8k_multiturn_grpo_w_interaction' \
     algorithm.adv_estimator=grpo \
     data.train_batch_size=$TRAIN_BATCH_SIZE \
     data.max_prompt_length=1024 \

examples/sglang_multiturn/search_r1_like/run_qwen2.5-3b_instruct_search_multiturn.sh

@@ -46,7 +46,7 @@ python3 -m verl.trainer.main_ppo \
     actor_rollout_ref.rollout.name=sglang_async \
     actor_rollout_ref.rollout.gpu_memory_utilization=0.5 \
     actor_rollout_ref.rollout.n=5 \
-    actor_rollout_ref.rollout.multi_turn.max_turns=2 \
+    actor_rollout_ref.rollout.multi_turn.max_assistant_turns=2 \
     actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=8 \
     actor_rollout_ref.ref.fsdp_config.param_offload=True \
     algorithm.use_kl_in_reward=False \

tests/workers/rollout/test_sglang_async_rollout_search_tools.py

@@ -190,7 +190,7 @@ def test_rollout_req_creation(self, mock_env, mock_engine, mock_sampling, search
     @patch.object(SGLangRollout, "_init_inference_engine", return_value=None)
     @patch.object(SGLangRollout, "_init_sampling_params", return_value=None)
     def test_over_size_case(self, mock_env, mock_engine, mock_sampling, search_rollout_config, qwen_tokenizer, qwen_model_config, search_data_proto, search_data):
-        search_rollout_config.multi_turn.max_turns = 1
+        search_rollout_config.multi_turn.max_assistant_turns = 1
         rollout = SGLangRollout(actor_module="", config=search_rollout_config, tokenizer=qwen_tokenizer, model_hf_config=qwen_model_config)
         req = rollout._preprocess_prompt_to_async_rollout_requests(search_data_proto, n=1)[0]
         req = MagicMock(wraps=req, spec=AsyncRolloutRequest)
@@ -232,7 +232,7 @@ def test_tool_call_basic_case(self, mock_sampling, mock_engine, mock_env, mock_e
         # Mock search tool execution to return predefined responses
         mock_execute.side_effect = [(msg, 0.0, {"status": "success"}) for msg in tool_return_array]
 
-        search_rollout_config.multi_turn.max_turns = 10
+        search_rollout_config.multi_turn.max_assistant_turns = 10
         rollout = SGLangRollout(actor_module="", config=search_rollout_config, tokenizer=qwen_tokenizer, model_hf_config=qwen_model_config)
 
         rollout._tool_map["search"].retrieval_service_url = "mock://dummy"
@@ -284,7 +284,7 @@ def test_tool_call_batch_case(self, mock_sampling, mock_engine, mock_env, mock_e
             (tool_return_array[1], 0.0, {"status": "success"}),
         ] * 100
 
-        search_rollout_config.multi_turn.max_turns = 10
+        search_rollout_config.multi_turn.max_assistant_turns = 10
         rollout = SGLangRollout(
             actor_module="",
             config=search_rollout_config,

tests/workers/rollout/test_sglang_async_rollout_sf_tools.py

@@ -246,7 +246,7 @@ def test_rollout_req_creation(self, mock_env, mock_engine, mock_sampling, sandbo
     @patch.object(SGLangRollout, "_init_inference_engine", return_value=None)
     @patch.object(SGLangRollout, "_init_sampling_params", return_value=None)
     def test_over_size_case(self, mock_env, mock_engine, mock_sampling, sandbox_fusion_rollout_config, qwen_tokenizer, qwen_model_config, sandbox_data_proto, sandbox_fusion_data):
-        sandbox_fusion_rollout_config.multi_turn.max_turns = 1
+        sandbox_fusion_rollout_config.multi_turn.max_assistant_turns = 1
         rollout = SGLangRollout(actor_module="", config=sandbox_fusion_rollout_config, tokenizer=qwen_tokenizer, model_hf_config=qwen_model_config)
         req = rollout._preprocess_prompt_to_async_rollout_requests(sandbox_data_proto, n=1)[0]
         req = MagicMock(wraps=req, spec=AsyncRolloutRequest)
@@ -283,7 +283,7 @@ def test_over_size_case(self, mock_env, mock_engine, mock_sampling, sandbox_fusi
     @patch.object(SGLangRollout, "_init_inference_engine", return_value=None)
     @patch.object(SGLangRollout, "_init_sampling_params", return_value=None)
     def test_tool_call_basic_case(self, mock_env, mock_engine, mock_sampling, sandbox_fusion_rollout_config, qwen_tokenizer, qwen_model_config, sandbox_data_proto, sandbox_fusion_data):
-        sandbox_fusion_rollout_config.multi_turn.max_turns = 10
+        sandbox_fusion_rollout_config.multi_turn.max_assistant_turns = 10
         rollout = SGLangRollout(actor_module="", config=sandbox_fusion_rollout_config, tokenizer=qwen_tokenizer, model_hf_config=qwen_model_config)
         self._tool_map["code_interpreter"].sandbox_fusion_url = sandbox_url
         req = rollout._preprocess_prompt_to_async_rollout_requests(sandbox_data_proto, n=1)[0]
@@ -327,7 +327,7 @@ def test_tool_call_basic_case(self, mock_env, mock_engine, mock_sampling, sandbo
     @patch.object(SGLangRollout, "_init_inference_engine", return_value=None)
     @patch.object(SGLangRollout, "_init_sampling_params", return_value=None)
     def test_tool_call_batch_case(self, mock_env, mock_engine, mock_sampling, sandbox_fusion_rollout_config, qwen_tokenizer, qwen_model_config, sandbox_data_proto, sandbox_fusion_data):
-        sandbox_fusion_rollout_config.multi_turn.max_turns = 10
+        sandbox_fusion_rollout_config.multi_turn.max_assistant_turns = 10
         rollout = SGLangRollout(actor_module="", config=sandbox_fusion_rollout_config, tokenizer=qwen_tokenizer, model_hf_config=qwen_model_config)
         self._tool_map["code_interpreter"].sandbox_fusion_url = sandbox_url
         req = rollout._preprocess_prompt_to_async_rollout_requests(sandbox_data_proto, n=1)[0]

verl/trainer/config/ppo_megatron_trainer.yaml

@@ -169,7 +169,7 @@ actor_rollout_ref:
       do_sample: False # default eager for validation
     multi_turn: 
       enable: False  # set to True for multi-turn tool interaction tasks; should set rollout.name to sglang as well
-      max_turns: null  # null for no limit (default max_length // 3)
+      max_assistant_turns: null  # null for no limit (default max_length // 3)
       tool_config_path: null  # null for no tool
       format: chatml  # chatml, more formats will be supported in the future
 

verl/workers/rollout/sglang_rollout/async_sglang_rollout.py

@@ -158,9 +158,11 @@ def _verify_config(self, model_hf_config):
         assert self.config.max_model_len >= self.config.prompt_length + self.config.response_length, f"""max_model_len should be greater than total sequence length (prompt_length + response_length): 
             {self.config.max_model_len} >= {self.config.prompt_length} + {self.config.response_length}"""
         assert model_hf_config.max_position_embeddings >= self.config.max_model_len, "model context length should be greater than total sequence length"
-        # currently max_turns stand for max number of tool calls
-        if self.config.multi_turn.max_turns is None:
-            self.config.multi_turn.max_turns = self.config.max_model_len // 3
+        # currently max_assistant_turns stand for max number of tool calls
+        if self.config.multi_turn.max_assistant_turns is None:
+            self.config.multi_turn.max_assistant_turns = self.config.max_model_len // 3
+        if self.config.multi_turn.max_user_turns is None:
+            self.config.multi_turn.max_user_turns = self.config.max_model_len // 3
 
     def _init_inference_engine(self, trust_remote_code, actor_module, port):
         # initialize the inference engine

verl/workers/rollout/sglang_rollout/sglang_rollout.py

@@ -224,9 +224,11 @@ def _verify_config(self, model_hf_config):
         assert self.config.max_model_len >= self.config.prompt_length + self.config.response_length, f"""max_model_len should be greater than total sequence length (prompt_length + response_length): 
             {self.config.max_model_len} >= {self.config.prompt_length} + {self.config.response_length}"""
         assert model_hf_config.max_position_embeddings >= self.config.max_model_len, "model context length should be greater than total sequence length"
-        # currently max_turns stand for max number of tool calls
-        if self.config.multi_turn.max_turns is None:
-            self.config.multi_turn.max_turns = self.config.max_model_len // 3
+        # currently max_assistant_turns stand for max number of tool calls
+        if self.config.multi_turn.max_assistant_turns is None:
+            self.config.multi_turn.max_assistant_turns = self.config.max_model_len // 3
+        if self.config.multi_turn.max_user_turns is None:
+            self.config.multi_turn.max_user_turns = self.config.max_model_len // 3
 
     def _init_inference_engine(self, trust_remote_code, actor_module, port):
         # initialize the inference engine
@@ -657,7 +659,7 @@ async def _async_rollout_a_request(
         user_turns = 0
         user_turn_rewards = []
 
-        while current_turns < self.config.multi_turn.max_turns:
+        while current_turns < self.config.multi_turn.max_assistant_turns:
             if _req.state == AsyncRolloutRequestStateEnum.PENDING:
                 await self._handle_pending_state(_req)
                 _req.state = AsyncRolloutRequestStateEnum.RUNNING