From 2fb11f18da78414c1eef69eef222486b7cba873e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 4 Jun 2026 17:02:38 +0000 Subject: [PATCH] test(core-utils): skip token_counter custom-tokenizer assertion when HF hub is unreachable test_tokenizers downloads Xenova/llama-3-tokenizer from the HuggingFace Hub via create_pretrained_tokenizer. On the CI runners the Hub keeps returning 429 Too Many Requests, which propagated into the blanket except and turned a third-party rate-limit into a hard pytest.fail. The same test already skips its llama2 differentiation assertion when the Hub is unreachable; this extends that exact handling to the custom tokenizer download so a HuggingFace outage/rate-limit no longer fails the suite while still failing on real assertion or logic errors. --- .../test_litellm/litellm_core_utils/test_token_counter.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py index 324bace0e96..92c070501b4 100644 --- a/tests/test_litellm/litellm_core_utils/test_token_counter.py +++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py @@ -200,7 +200,12 @@ def test_tokenizers(): model="meta-llama/llama-3-70b-instruct", text=sample_text ) - llama3_tokenizer = create_pretrained_tokenizer("Xenova/llama-3-tokenizer") + try: + llama3_tokenizer = create_pretrained_tokenizer("Xenova/llama-3-tokenizer") + except Exception as e: + pytest.skip( + f"custom tokenizer download failed (HF hub unreachable): {e}" + ) llama3_tokens_2 = token_counter( custom_tokenizer=llama3_tokenizer, text=sample_text )