From 6183c5f816e1090ab2ebddba55f12adaafaa9881 Mon Sep 17 00:00:00 2001 From: qqiao Date: Fri, 8 Aug 2025 02:59:55 -0700 Subject: [PATCH 1/5] Waive failed tests for main Signed-off-by: qqiao --- tests/integration/test_lists/waives.txt | 7 +++++++ tests/unittest/llmapi/test_llm_kv_cache_events.py | 1 + tests/unittest/trt/model/test_nemotron_nas.py | 2 ++ 3 files changed, 10 insertions(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 15b8279a21b..14bb3a7824c 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -287,3 +287,10 @@ test_e2e.py::test_ptp_quickstart_multimodal[llava-v1.6-mistral-7b-llava-v1.6-mis accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=True] SKIP (https://nvbugs/5433545) test_e2e.py::test_ptp_quickstart_multimodal[llava-v1.6-mistral-7b-llava-v1.6-mistral-7b-hf-image-False] SKIP (https://nvbugs/5444095) test_e2e.py::test_ptp_quickstart_multimodal[NVILA-8B-FP16-vila/NVILA-8B-image-False] SKIP (https://nvbugs/5444095) +examples/test_nemotron_nas.py::test_nemotron_nas_summary_1gpu[DeciLM-7B] SKIP (https://nvbugs/5444636) +accuracy/test_cli_flow.py::TestLongAlpaca7B::test_multiblock_aggressive SKIP (https://nvbugs/5444627) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus_online_eplb[mtp_nextn=2] SKIP (https://nvbugs/5444687) +accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus_online_eplb[fp8kv=True] SKIP (https://nvbugs/5444687) +accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency] SKIP (https://nvbugs/5445466) +accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_trtllmgen] SKIP (https://nvbugs/5445466) +disaggregated/test_disaggregated.py::test_disaggregated_ctxpp4_genpp4[llama-3.1-8b] SKIP (https://nvbugs/5445642) diff --git a/tests/unittest/llmapi/test_llm_kv_cache_events.py b/tests/unittest/llmapi/test_llm_kv_cache_events.py index 48665c3e25a..9297e1bb2e7 100644 --- a/tests/unittest/llmapi/test_llm_kv_cache_events.py +++ b/tests/unittest/llmapi/test_llm_kv_cache_events.py @@ -231,6 +231,7 @@ def check_events(llm, assert not llm.get_kv_cache_events(5) +@pytest.mark.skip(reason="https://nvbugs/5445001") def test_llm_kv_events_api(): llm = create_llm() sampling_params = SamplingParams(max_tokens=6, diff --git a/tests/unittest/trt/model/test_nemotron_nas.py b/tests/unittest/trt/model/test_nemotron_nas.py index 07e1c1939db..30d2afc1886 100644 --- a/tests/unittest/trt/model/test_nemotron_nas.py +++ b/tests/unittest/trt/model/test_nemotron_nas.py @@ -380,6 +380,7 @@ def get_loader_test_cases(): return test_cases + @pytest.mark.skip(reason="https://nvbugs/5444611") @parameterized.expand(get_loader_test_cases, name_func=unittest_name_func) def test_allclose_to_hf(self, hf_model_dir: str, params: TestParams): hf_model = transformers.AutoModelForCausalLM.from_pretrained( @@ -819,6 +820,7 @@ def test_convert_config_from_hf(self, ckpt_path: Optional[str], # verify config is valid enough for model creation DeciLMForCausalLM(config) + @pytest.mark.skip(reason="https://nvbugs/5444611") @parameterized.expand( itertools.product( os.listdir( From 23a3e7e4fbee4a7ec3310b4c0809c31f967f5844 Mon Sep 17 00:00:00 2001 From: qqiao Date: Fri, 8 Aug 2025 03:06:10 -0700 Subject: [PATCH 2/5] correct a bugid in waive file Signed-off-by: qqiao --- tests/integration/test_lists/waives.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 14bb3a7824c..244032f59cb 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -280,7 +280,7 @@ accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen examples/test_gemma.py::test_hf_gemma_fp8_base_bf16_multi_lora[gemma-2-9b-it] SKIP (https://nvbugs/5434451) examples/test_gemma.py::test_hf_gemma_fp8_base_bf16_multi_lora[gemma-2-27b-it] SKIP (https://nvbugs/5434451) examples/test_gemma.py::test_hf_gemma_fp8_base_bf16_multi_lora[gemma-3-1b-it] SKIP (https://nvbugs/5434451) -accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_trtllm_eagle3] SKIP (https://nvbugs/5437384) +accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4[latency_moe_trtllm_eagle3] SKIP (https://nvbugs/5437405) accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4 SKIP (https://nvbugs/5440241) test_e2e.py::test_ptp_quickstart_multimodal[NVILA-8B-FP16-vila/NVILA-8B-image-False] SKIP (https://nvbugs/5444060) test_e2e.py::test_ptp_quickstart_multimodal[llava-v1.6-mistral-7b-llava-v1.6-mistral-7b-hf-image-False] SKIP (https://nvbugs/5444060) From 0a954d5983fc97f08ebb6fb08fb9aaa8ac8b5516 Mon Sep 17 00:00:00 2001 From: qqiao Date: Sat, 9 Aug 2025 07:26:55 -0700 Subject: [PATCH 3/5] Waive a missing case and fix ut waive Signed-off-by: qqiao --- tests/integration/test_lists/waives.txt | 1 + tests/unittest/trt/model/test_nemotron_nas.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 244032f59cb..d6463637208 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -294,3 +294,4 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus_online_ep accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency] SKIP (https://nvbugs/5445466) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_trtllmgen] SKIP (https://nvbugs/5445466) disaggregated/test_disaggregated.py::test_disaggregated_ctxpp4_genpp4[llama-3.1-8b] SKIP (https://nvbugs/5445642) +examples/test_qwen2audio.py::test_llm_qwen2audio_single_gpu[qwen2_audio_7b_instruct] SKIP (https://nvbugs/5447530) diff --git a/tests/unittest/trt/model/test_nemotron_nas.py b/tests/unittest/trt/model/test_nemotron_nas.py index 30d2afc1886..5a99291ddf4 100644 --- a/tests/unittest/trt/model/test_nemotron_nas.py +++ b/tests/unittest/trt/model/test_nemotron_nas.py @@ -380,9 +380,9 @@ def get_loader_test_cases(): return test_cases - @pytest.mark.skip(reason="https://nvbugs/5444611") @parameterized.expand(get_loader_test_cases, name_func=unittest_name_func) def test_allclose_to_hf(self, hf_model_dir: str, params: TestParams): + self.skipTest(f"https://nvbugs/5444611") hf_model = transformers.AutoModelForCausalLM.from_pretrained( hf_model_dir, trust_remote_code=True, @@ -820,7 +820,6 @@ def test_convert_config_from_hf(self, ckpt_path: Optional[str], # verify config is valid enough for model creation DeciLMForCausalLM(config) - @pytest.mark.skip(reason="https://nvbugs/5444611") @parameterized.expand( itertools.product( os.listdir( @@ -829,6 +828,7 @@ def test_convert_config_from_hf(self, ckpt_path: Optional[str], def test_convert_model_from_hf(self, model_dir: Optional[str], preloaded: bool, tp_size: int, pp_size: int, dtype: str) -> None: + self.skipTest(f"https://nvbugs/5444611") ckpt_path = Path(llm_models_root(check=True), "nvsmall/tests", model_dir) From 99a0423f338822b5b6f6cdd5b2a9e875abf2b5d8 Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Sun, 10 Aug 2025 10:58:44 +0800 Subject: [PATCH 4/5] Update test_nemotron_nas.py Signed-off-by: Yanchao Lu --- tests/unittest/trt/model/test_nemotron_nas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unittest/trt/model/test_nemotron_nas.py b/tests/unittest/trt/model/test_nemotron_nas.py index 5a99291ddf4..30d2afc1886 100644 --- a/tests/unittest/trt/model/test_nemotron_nas.py +++ b/tests/unittest/trt/model/test_nemotron_nas.py @@ -380,9 +380,9 @@ def get_loader_test_cases(): return test_cases + @pytest.mark.skip(reason="https://nvbugs/5444611") @parameterized.expand(get_loader_test_cases, name_func=unittest_name_func) def test_allclose_to_hf(self, hf_model_dir: str, params: TestParams): - self.skipTest(f"https://nvbugs/5444611") hf_model = transformers.AutoModelForCausalLM.from_pretrained( hf_model_dir, trust_remote_code=True, @@ -820,6 +820,7 @@ def test_convert_config_from_hf(self, ckpt_path: Optional[str], # verify config is valid enough for model creation DeciLMForCausalLM(config) + @pytest.mark.skip(reason="https://nvbugs/5444611") @parameterized.expand( itertools.product( os.listdir( @@ -828,7 +829,6 @@ def test_convert_config_from_hf(self, ckpt_path: Optional[str], def test_convert_model_from_hf(self, model_dir: Optional[str], preloaded: bool, tp_size: int, pp_size: int, dtype: str) -> None: - self.skipTest(f"https://nvbugs/5444611") ckpt_path = Path(llm_models_root(check=True), "nvsmall/tests", model_dir) From 270d91275f42ab1c3f0b8f823fcf8065002469b3 Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Sun, 10 Aug 2025 11:27:48 +0800 Subject: [PATCH 5/5] Update test_nemotron_nas.py Signed-off-by: Yanchao Lu --- tests/unittest/trt/model/test_nemotron_nas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unittest/trt/model/test_nemotron_nas.py b/tests/unittest/trt/model/test_nemotron_nas.py index 30d2afc1886..5a99291ddf4 100644 --- a/tests/unittest/trt/model/test_nemotron_nas.py +++ b/tests/unittest/trt/model/test_nemotron_nas.py @@ -380,9 +380,9 @@ def get_loader_test_cases(): return test_cases - @pytest.mark.skip(reason="https://nvbugs/5444611") @parameterized.expand(get_loader_test_cases, name_func=unittest_name_func) def test_allclose_to_hf(self, hf_model_dir: str, params: TestParams): + self.skipTest(f"https://nvbugs/5444611") hf_model = transformers.AutoModelForCausalLM.from_pretrained( hf_model_dir, trust_remote_code=True, @@ -820,7 +820,6 @@ def test_convert_config_from_hf(self, ckpt_path: Optional[str], # verify config is valid enough for model creation DeciLMForCausalLM(config) - @pytest.mark.skip(reason="https://nvbugs/5444611") @parameterized.expand( itertools.product( os.listdir( @@ -829,6 +828,7 @@ def test_convert_config_from_hf(self, ckpt_path: Optional[str], def test_convert_model_from_hf(self, model_dir: Optional[str], preloaded: bool, tp_size: int, pp_size: int, dtype: str) -> None: + self.skipTest(f"https://nvbugs/5444611") ckpt_path = Path(llm_models_root(check=True), "nvsmall/tests", model_dir)