From 2d36596e8b650963f389dc7ff81b09f061c47255 Mon Sep 17 00:00:00 2001
From: user <70670632+stuxf@users.noreply.github.com>
Date: Mon, 4 May 2026 11:47:32 -0700
Subject: [PATCH] fix: preserve tokenizer decode round trips

---
 litellm/utils.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/litellm/utils.py b/litellm/utils.py
index 8c1b6452ced..ec198616f05 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -2245,10 +2245,32 @@ def encode(model="", text="", custom_tokenizer: Optional[dict] = None):
 
 def decode(model="", tokens: List[int] = [], custom_tokenizer: Optional[dict] = None):
     tokenizer_json = custom_tokenizer or _select_tokenizer(model=model)
+    if tokenizer_json["type"] == "huggingface_tokenizer":
+        tokens = _strip_huggingface_special_token_ids(
+            tokenizer_json["tokenizer"], tokens
+        )
     dec = tokenizer_json["tokenizer"].decode(tokens)
     return dec
 
 
+def _strip_huggingface_special_token_ids(
+    tokenizer: Tokenizer, tokens: List[int]
+) -> List[int]:
+    try:
+        added_tokens_decoder = tokenizer.get_added_tokens_decoder()
+    except Exception:
+        return tokens
+
+    special_token_ids = {
+        token_id
+        for token_id, added_token in added_tokens_decoder.items()
+        if getattr(added_token, "special", False)
+    }
+    if not special_token_ids:
+        return tokens
+    return [token for token in tokens if token not in special_token_ids]
+
+
 def create_pretrained_tokenizer(
     identifier: str, revision="main", auth_token: Optional[str] = None
 ):