From e9328bfa3aa820c4af25e5bae4da9f5cee6557a3 Mon Sep 17 00:00:00 2001
From: Fabricio Ceschin <fceschin@todyl.com>
Date: Mon, 19 May 2025 14:48:41 -0400
Subject: [PATCH 01/19] Adding langfuse usage details for cached tokens

---
 litellm/integrations/langfuse/langfuse.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py
index d0472ee6383..ccc072149cc 100644
--- a/litellm/integrations/langfuse/langfuse.py
+++ b/litellm/integrations/langfuse/langfuse.py
@@ -638,6 +638,7 @@ class LangFuseLogger:
 
             generation_id = None
             usage = None
+            usage_details = None
             if response_obj is not None:
                 if (
                     hasattr(response_obj, "id")
@@ -654,6 +655,13 @@ class LangFuseLogger:
                         "completion_tokens": _usage_obj.completion_tokens,
                         "total_cost": cost if self._supports_costs() else None,
                     }
+                    usage_details = {
+                        "input": _usage_obj.prompt_tokens,
+                        "output": _usage_obj.completion_tokens,
+                        "cache_creation_input_tokens": _usage_obj.get('cache_creation_input_tokens', 0),
+                        "cache_read_input_tokens": _usage_obj.get('cache_read_input_tokens', 0)
+                    }
+
             generation_name = clean_metadata.pop("generation_name", None)
             if generation_name is None:
                 # if `generation_name` is None, use sensible default values
@@ -686,6 +694,7 @@ class LangFuseLogger:
                 "input": input if not mask_input else "redacted-by-litellm",
                 "output": output if not mask_output else "redacted-by-litellm",
                 "usage": usage,
+                "usage_details": usage_details,
                 "metadata": log_requester_metadata(clean_metadata),
                 "level": level,
                 "version": clean_metadata.pop("version", None),

From d47ad69a1b2d006fbc15acea12bc47cc9d330bc1 Mon Sep 17 00:00:00 2001
From: Fabricio Ceschin <fceschin@todyl.com>
Date: Wed, 21 May 2025 10:43:08 -0400
Subject: [PATCH 02/19] Added typed dict for LangfuseUsageDetails

---
 litellm/integrations/langfuse/langfuse.py | 2 +-
 litellm/types/integrations/langfuse.py    | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py
index f4d1a1f79c4..17e15e0b03c 100644
--- a/litellm/integrations/langfuse/langfuse.py
+++ b/litellm/integrations/langfuse/langfuse.py
@@ -659,7 +659,7 @@ class LangFuseLogger:
                         "completion_tokens": _usage_obj.completion_tokens,
                         "total_cost": cost if self._supports_costs() else None,
                     }
-                    usage_details = {
+                    usage_details : Optional[LangfuseUsageDetails] = {
                         "input": _usage_obj.prompt_tokens,
                         "output": _usage_obj.completion_tokens,
                         "cache_creation_input_tokens": _usage_obj.get('cache_creation_input_tokens', 0),
diff --git a/litellm/types/integrations/langfuse.py b/litellm/types/integrations/langfuse.py
index ecf42d8cd47..c19cd98d162 100644
--- a/litellm/types/integrations/langfuse.py
+++ b/litellm/types/integrations/langfuse.py
@@ -5,3 +5,10 @@ class LangfuseLoggingConfig(TypedDict):
     langfuse_secret: Optional[str]
     langfuse_public_key: Optional[str]
     langfuse_host: Optional[str]
+
+
+class LangfuseUsageDetails(TypedDict):
+    input: Optional[int]
+    output: Optional[int]
+    cache_creation_input_tokens: Optional[int]
+    cache_read_input_tokens: Optional[int]

From a30fd5c2ad59682d250493996e55327abd5e645b Mon Sep 17 00:00:00 2001
From: Fabricio Ceschin <fceschin@todyl.com>
Date: Wed, 21 May 2025 10:51:59 -0400
Subject: [PATCH 03/19] Fixing lint errors with LangfuseUsageDetails

---
 litellm/integrations/langfuse/langfuse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py
index 17e15e0b03c..001e79cf802 100644
--- a/litellm/integrations/langfuse/langfuse.py
+++ b/litellm/integrations/langfuse/langfuse.py
@@ -642,7 +642,7 @@ class LangFuseLogger:
 
             generation_id = None
             usage = None
-            usage_details = None
+            usage_details : Optional[LangfuseUsageDetails] = None
             if response_obj is not None:
                 if (
                     hasattr(response_obj, "id")

From 90c54d688a4b290312c79a5c50cfbb227aa09670 Mon Sep 17 00:00:00 2001
From: Fabricio Ceschin <fceschin@todyl.com>
Date: Wed, 21 May 2025 11:03:14 -0400
Subject: [PATCH 04/19] Fixing lint errors with LangfuseUsageDetails again

---
 litellm/integrations/langfuse/langfuse.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py
index 001e79cf802..a98771387c0 100644
--- a/litellm/integrations/langfuse/langfuse.py
+++ b/litellm/integrations/langfuse/langfuse.py
@@ -659,7 +659,7 @@ class LangFuseLogger:
                         "completion_tokens": _usage_obj.completion_tokens,
                         "total_cost": cost if self._supports_costs() else None,
                     }
-                    usage_details : Optional[LangfuseUsageDetails] = {
+                    usage_details = {
                         "input": _usage_obj.prompt_tokens,
                         "output": _usage_obj.completion_tokens,
                         "cache_creation_input_tokens": _usage_obj.get('cache_creation_input_tokens', 0),

From bd0369dd887e3fb9f630c6e25d6962f42635a0b5 Mon Sep 17 00:00:00 2001
From: Fabricio Ceschin <fceschin@todyl.com>
Date: Wed, 21 May 2025 13:35:21 -0400
Subject: [PATCH 05/19] Using typed dict in usage_details

---
 litellm/integrations/langfuse/langfuse.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py
index a98771387c0..f116eb8041b 100644
--- a/litellm/integrations/langfuse/langfuse.py
+++ b/litellm/integrations/langfuse/langfuse.py
@@ -642,7 +642,7 @@ class LangFuseLogger:
 
             generation_id = None
             usage = None
-            usage_details : Optional[LangfuseUsageDetails] = None
+            usage_details = None
             if response_obj is not None:
                 if (
                     hasattr(response_obj, "id")
@@ -659,12 +659,10 @@ class LangFuseLogger:
                         "completion_tokens": _usage_obj.completion_tokens,
                         "total_cost": cost if self._supports_costs() else None,
                     }
-                    usage_details = {
-                        "input": _usage_obj.prompt_tokens,
-                        "output": _usage_obj.completion_tokens,
-                        "cache_creation_input_tokens": _usage_obj.get('cache_creation_input_tokens', 0),
-                        "cache_read_input_tokens": _usage_obj.get('cache_read_input_tokens', 0)
-                    }
+                    usage_details = LangfuseUsageDetails(input=_usage_obj.prompt_tokens,
+                                                        output=_usage_obj.completion_tokens,
+                                                        cache_creation_input_tokens=_usage_obj.get('cache_creation_input_tokens', 0),
+                                                        cache_read_input_tokens=_usage_obj.get('cache_read_input_tokens', 0))
 
             generation_name = clean_metadata.pop("generation_name", None)
             if generation_name is None:

From 9d73f1e607be57c65c6d8061f060c15c7e92a21d Mon Sep 17 00:00:00 2001
From: Fabricio Ceschin <fceschin@todyl.com>
Date: Wed, 21 May 2025 21:23:49 -0400
Subject: [PATCH 06/19] Added unit tests for langfuse usage details integration

---
 tests/litellm/integrations/test_langfuse.py | 218 ++++++++++++++++++++
 1 file changed, 218 insertions(+)
 create mode 100644 tests/litellm/integrations/test_langfuse.py

diff --git a/tests/litellm/integrations/test_langfuse.py b/tests/litellm/integrations/test_langfuse.py
new file mode 100644
index 00000000000..26874d9ba21
--- /dev/null
+++ b/tests/litellm/integrations/test_langfuse.py
@@ -0,0 +1,218 @@
+import unittest
+from unittest.mock import patch, MagicMock
+import sys
+import os
+import datetime
+
+sys.path.insert(0, os.path.abspath("../.."))
+from litellm.integrations.langfuse.langfuse import LangFuseLogger
+# Import LangfuseUsageDetails directly from the module where it's defined
+from litellm.types.integrations.langfuse import *
+
+
+
+class TestLangfuseUsageDetails(unittest.TestCase):
+    
+    def setUp(self):
+        # Set up environment variables for testing
+        self.env_patcher = patch.dict('os.environ', {
+            'LANGFUSE_SECRET_KEY': 'test-secret-key',
+            'LANGFUSE_PUBLIC_KEY': 'test-public-key',
+            'LANGFUSE_HOST': 'https://test.langfuse.com'
+        })
+        self.env_patcher.start()
+        
+        # Create mock objects
+        self.mock_langfuse_client = MagicMock()
+        self.mock_langfuse_trace = MagicMock()
+        self.mock_langfuse_generation = MagicMock()
+        
+        # Setup the trace and generation chain
+        self.mock_langfuse_trace.generation.return_value = self.mock_langfuse_generation
+        self.mock_langfuse_client.trace.return_value = self.mock_langfuse_trace
+        
+        # Mock the langfuse module that's imported locally in methods
+        self.langfuse_module_patcher = patch.dict('sys.modules', {'langfuse': MagicMock()})
+        self.mock_langfuse_module = self.langfuse_module_patcher.start()
+        
+        # Create a mock for the langfuse module with version
+        self.mock_langfuse = MagicMock()
+        self.mock_langfuse.version = MagicMock()
+        self.mock_langfuse.version.__version__ = "3.0.0"  # Set a version that supports all features
+        
+        # Mock the Langfuse class
+        self.mock_langfuse_class = MagicMock()
+        self.mock_langfuse_class.return_value = self.mock_langfuse_client
+        
+        # Set up the sys.modules['langfuse'] mock
+        sys.modules['langfuse'] = self.mock_langfuse
+        sys.modules['langfuse'].Langfuse = self.mock_langfuse_class
+        
+        # Mock the Langfuse client
+        self.mock_langfuse_client = MagicMock()
+        self.mock_langfuse_trace = MagicMock()
+        self.mock_langfuse_generation = MagicMock()
+        
+        # Setup the trace and generation chain
+        self.mock_langfuse_trace.generation.return_value = self.mock_langfuse_generation
+        self.mock_langfuse_client.trace.return_value = self.mock_langfuse_trace
+        
+        # Mock the Langfuse class
+        self.mock_langfuse_class = MagicMock()
+        self.mock_langfuse_class.return_value = self.mock_langfuse_client
+        self.mock_langfuse.Langfuse = self.mock_langfuse_class
+        
+        # Create the logger
+        self.logger = LangFuseLogger()
+        
+        # Add the log_event_on_langfuse method to the instance
+        def log_event_on_langfuse(self, kwargs, response_obj, start_time=None, end_time=None, user_id=None, level="DEFAULT", status_message=None):
+            # This implementation calls _log_langfuse_v2 directly
+            return self._log_langfuse_v2(
+                user_id=user_id,
+                metadata=kwargs.get("litellm_params", {}).get("metadata", {}),
+                litellm_params=kwargs.get("litellm_params", {}),
+                output=None,
+                start_time=start_time,
+                end_time=end_time,
+                kwargs=kwargs,
+                optional_params=kwargs.get("optional_params", {}),
+                input=None,
+                response_obj=response_obj,
+                level=level,
+                litellm_call_id=kwargs.get("litellm_call_id", None),
+                print_verbose=True  # Add the missing parameter
+            )
+        
+        # Bind the method to the instance
+        import types
+        self.logger.log_event_on_langfuse = types.MethodType(log_event_on_langfuse, self.logger)
+        
+        # Make sure _is_langfuse_v2 returns True
+        def mock_is_langfuse_v2(self):
+            return True
+        
+        self.logger._is_langfuse_v2 = types.MethodType(mock_is_langfuse_v2, self.logger)
+    
+    def tearDown(self):
+        self.env_patcher.stop()
+        self.langfuse_module_patcher.stop()
+    
+    def test_langfuse_usage_details_type(self):
+        """Test that LangfuseUsageDetails TypedDict is properly defined with the correct fields"""
+        # Create an instance of LangfuseUsageDetails
+        usage_details: LangfuseUsageDetails = {
+            "input": 10,
+            "output": 20,
+            "cache_creation_input_tokens": 5,
+            "cache_read_input_tokens": 3
+        }
+        
+        # Verify all fields are present
+        self.assertEqual(usage_details["input"], 10)
+        self.assertEqual(usage_details["output"], 20)
+        self.assertEqual(usage_details["cache_creation_input_tokens"], 5)
+        self.assertEqual(usage_details["cache_read_input_tokens"], 3)
+        
+        # Test with all fields (all fields are required in TypedDict by default)
+        minimal_usage_details: LangfuseUsageDetails = {
+            "input": 10,
+            "output": 20,
+            "cache_creation_input_tokens": 0,
+            "cache_read_input_tokens": 0
+        }
+        
+        self.assertEqual(minimal_usage_details["input"], 10)
+        self.assertEqual(minimal_usage_details["output"], 20)
+        
+    def test_log_langfuse_v2_usage_details(self):
+        """Test that usage_details in _log_langfuse_v2 is correctly typed and assigned"""
+        # Create a mock response object with usage information
+        response_obj = MagicMock()
+        response_obj.usage = MagicMock()
+        response_obj.usage.prompt_tokens = 15
+        response_obj.usage.completion_tokens = 25
+        
+        # Add the cache token attributes using get method
+        def mock_get(key, default=None):
+            if key == 'cache_creation_input_tokens':
+                return 7
+            elif key == 'cache_read_input_tokens':
+                return 4
+            return default
+        
+        response_obj.usage.get = mock_get
+        
+        # Create kwargs for the log_event method
+        kwargs = {
+            "model": "gpt-4",
+            "messages": [{"role": "user", "content": "Hello"}],
+            "litellm_params": {"metadata": {}}
+        }
+        
+        # Create start and end times
+        start_time = datetime.datetime.now()
+        end_time = start_time + datetime.timedelta(seconds=1)
+        
+        # Call the log_event method
+        with patch.object(self.logger, '_log_langfuse_v2') as mock_log_langfuse_v2:
+            self.logger.log_event_on_langfuse(
+                kwargs=kwargs,
+                response_obj=response_obj,
+                start_time=start_time,
+                end_time=end_time
+            )
+            
+            # Check if _log_langfuse_v2 was called
+            mock_log_langfuse_v2.assert_called_once()
+            
+            # Get the arguments passed to _log_langfuse_v2
+            call_args = mock_log_langfuse_v2.call_args[1]
+            
+            # Verify response_obj was passed correctly
+            self.assertEqual(call_args["response_obj"], response_obj)
+    
+    def test_langfuse_usage_details_optional_fields(self):
+        """Test that LangfuseUsageDetails fields are properly defined as Optional"""
+        # Create an instance with None values for optional fields
+        usage_details: LangfuseUsageDetails = {
+            "input": 10,
+            "output": 20,
+            "cache_creation_input_tokens": None,
+            "cache_read_input_tokens": None
+        }
+        
+        # Verify fields can be None
+        self.assertEqual(usage_details["input"], 10)
+        self.assertEqual(usage_details["output"], 20)
+        self.assertIsNone(usage_details["cache_creation_input_tokens"])
+        self.assertIsNone(usage_details["cache_read_input_tokens"])
+    
+    def test_langfuse_usage_details_structure(self):
+        """Test that LangfuseUsageDetails has the correct structure as defined in the commit"""
+        # This test directly verifies the structure of the TypedDict
+        # without relying on the LangFuseLogger class
+        
+        # Create a dictionary that matches the LangfuseUsageDetails structure
+        usage_details = {
+            "input": 15,
+            "output": 25,
+            "cache_creation_input_tokens": 7,
+            "cache_read_input_tokens": 4
+        }
+        
+        # Verify the structure matches what we expect
+        self.assertIn("input", usage_details)
+        self.assertIn("output", usage_details)
+        self.assertIn("cache_creation_input_tokens", usage_details)
+        self.assertIn("cache_read_input_tokens", usage_details)
+        
+        # Verify the values
+        self.assertEqual(usage_details["input"], 15)
+        self.assertEqual(usage_details["output"], 25)
+        self.assertEqual(usage_details["cache_creation_input_tokens"], 7)
+        self.assertEqual(usage_details["cache_read_input_tokens"], 4)
+
+
+if __name__ == "__main__":
+    unittest.main()

From 118d15c1c61e26daf0362bc08e13e28a4eafc4f6 Mon Sep 17 00:00:00 2001
From: Fabricio Ceschin <fceschin@todyl.com>
Date: Wed, 21 May 2025 21:37:23 -0400
Subject: [PATCH 07/19] Improving langfuse usage details unit test

---
 tests/litellm/integrations/test_langfuse.py | 76 ++++++++++-----------
 1 file changed, 35 insertions(+), 41 deletions(-)

diff --git a/tests/litellm/integrations/test_langfuse.py b/tests/litellm/integrations/test_langfuse.py
index 26874d9ba21..2aa363e5f94 100644
--- a/tests/litellm/integrations/test_langfuse.py
+++ b/tests/litellm/integrations/test_langfuse.py
@@ -9,10 +9,8 @@ from litellm.integrations.langfuse.langfuse import LangFuseLogger
 # Import LangfuseUsageDetails directly from the module where it's defined
 from litellm.types.integrations.langfuse import *
 
-
-
 class TestLangfuseUsageDetails(unittest.TestCase):
-    
+
     def setUp(self):
         # Set up environment variables for testing
         self.env_patcher = patch.dict('os.environ', {
@@ -21,50 +19,50 @@ class TestLangfuseUsageDetails(unittest.TestCase):
             'LANGFUSE_HOST': 'https://test.langfuse.com'
         })
         self.env_patcher.start()
-        
+
         # Create mock objects
         self.mock_langfuse_client = MagicMock()
         self.mock_langfuse_trace = MagicMock()
         self.mock_langfuse_generation = MagicMock()
-        
+
         # Setup the trace and generation chain
         self.mock_langfuse_trace.generation.return_value = self.mock_langfuse_generation
         self.mock_langfuse_client.trace.return_value = self.mock_langfuse_trace
-        
+
         # Mock the langfuse module that's imported locally in methods
         self.langfuse_module_patcher = patch.dict('sys.modules', {'langfuse': MagicMock()})
         self.mock_langfuse_module = self.langfuse_module_patcher.start()
-        
+
         # Create a mock for the langfuse module with version
         self.mock_langfuse = MagicMock()
         self.mock_langfuse.version = MagicMock()
         self.mock_langfuse.version.__version__ = "3.0.0"  # Set a version that supports all features
-        
+
         # Mock the Langfuse class
         self.mock_langfuse_class = MagicMock()
         self.mock_langfuse_class.return_value = self.mock_langfuse_client
-        
+
         # Set up the sys.modules['langfuse'] mock
         sys.modules['langfuse'] = self.mock_langfuse
         sys.modules['langfuse'].Langfuse = self.mock_langfuse_class
-        
+
         # Mock the Langfuse client
         self.mock_langfuse_client = MagicMock()
         self.mock_langfuse_trace = MagicMock()
         self.mock_langfuse_generation = MagicMock()
-        
+
         # Setup the trace and generation chain
         self.mock_langfuse_trace.generation.return_value = self.mock_langfuse_generation
         self.mock_langfuse_client.trace.return_value = self.mock_langfuse_trace
-        
+
         # Mock the Langfuse class
         self.mock_langfuse_class = MagicMock()
         self.mock_langfuse_class.return_value = self.mock_langfuse_client
         self.mock_langfuse.Langfuse = self.mock_langfuse_class
-        
+
         # Create the logger
         self.logger = LangFuseLogger()
-        
+
         # Add the log_event_on_langfuse method to the instance
         def log_event_on_langfuse(self, kwargs, response_obj, start_time=None, end_time=None, user_id=None, level="DEFAULT", status_message=None):
             # This implementation calls _log_langfuse_v2 directly
@@ -83,21 +81,21 @@ class TestLangfuseUsageDetails(unittest.TestCase):
                 litellm_call_id=kwargs.get("litellm_call_id", None),
                 print_verbose=True  # Add the missing parameter
             )
-        
+
         # Bind the method to the instance
         import types
         self.logger.log_event_on_langfuse = types.MethodType(log_event_on_langfuse, self.logger)
-        
+
         # Make sure _is_langfuse_v2 returns True
         def mock_is_langfuse_v2(self):
             return True
-        
+
         self.logger._is_langfuse_v2 = types.MethodType(mock_is_langfuse_v2, self.logger)
-    
+
     def tearDown(self):
         self.env_patcher.stop()
         self.langfuse_module_patcher.stop()
-    
+
     def test_langfuse_usage_details_type(self):
         """Test that LangfuseUsageDetails TypedDict is properly defined with the correct fields"""
         # Create an instance of LangfuseUsageDetails
@@ -107,13 +105,13 @@ class TestLangfuseUsageDetails(unittest.TestCase):
             "cache_creation_input_tokens": 5,
             "cache_read_input_tokens": 3
         }
-        
+
         # Verify all fields are present
         self.assertEqual(usage_details["input"], 10)
         self.assertEqual(usage_details["output"], 20)
         self.assertEqual(usage_details["cache_creation_input_tokens"], 5)
         self.assertEqual(usage_details["cache_read_input_tokens"], 3)
-        
+
         # Test with all fields (all fields are required in TypedDict by default)
         minimal_usage_details: LangfuseUsageDetails = {
             "input": 10,
@@ -121,10 +119,10 @@ class TestLangfuseUsageDetails(unittest.TestCase):
             "cache_creation_input_tokens": 0,
             "cache_read_input_tokens": 0
         }
-        
+
         self.assertEqual(minimal_usage_details["input"], 10)
         self.assertEqual(minimal_usage_details["output"], 20)
-        
+
     def test_log_langfuse_v2_usage_details(self):
         """Test that usage_details in _log_langfuse_v2 is correctly typed and assigned"""
         # Create a mock response object with usage information
@@ -132,7 +130,7 @@ class TestLangfuseUsageDetails(unittest.TestCase):
         response_obj.usage = MagicMock()
         response_obj.usage.prompt_tokens = 15
         response_obj.usage.completion_tokens = 25
-        
+
         # Add the cache token attributes using get method
         def mock_get(key, default=None):
             if key == 'cache_creation_input_tokens':
@@ -140,20 +138,20 @@ class TestLangfuseUsageDetails(unittest.TestCase):
             elif key == 'cache_read_input_tokens':
                 return 4
             return default
-        
+
         response_obj.usage.get = mock_get
-        
+
         # Create kwargs for the log_event method
         kwargs = {
             "model": "gpt-4",
             "messages": [{"role": "user", "content": "Hello"}],
             "litellm_params": {"metadata": {}}
         }
-        
+
         # Create start and end times
         start_time = datetime.datetime.now()
         end_time = start_time + datetime.timedelta(seconds=1)
-        
+
         # Call the log_event method
         with patch.object(self.logger, '_log_langfuse_v2') as mock_log_langfuse_v2:
             self.logger.log_event_on_langfuse(
@@ -162,16 +160,16 @@ class TestLangfuseUsageDetails(unittest.TestCase):
                 start_time=start_time,
                 end_time=end_time
             )
-            
+
             # Check if _log_langfuse_v2 was called
             mock_log_langfuse_v2.assert_called_once()
-            
+
             # Get the arguments passed to _log_langfuse_v2
             call_args = mock_log_langfuse_v2.call_args[1]
-            
+
             # Verify response_obj was passed correctly
             self.assertEqual(call_args["response_obj"], response_obj)
-    
+
     def test_langfuse_usage_details_optional_fields(self):
         """Test that LangfuseUsageDetails fields are properly defined as Optional"""
         # Create an instance with None values for optional fields
@@ -181,18 +179,18 @@ class TestLangfuseUsageDetails(unittest.TestCase):
             "cache_creation_input_tokens": None,
             "cache_read_input_tokens": None
         }
-        
+
         # Verify fields can be None
         self.assertEqual(usage_details["input"], 10)
         self.assertEqual(usage_details["output"], 20)
         self.assertIsNone(usage_details["cache_creation_input_tokens"])
         self.assertIsNone(usage_details["cache_read_input_tokens"])
-    
+
     def test_langfuse_usage_details_structure(self):
         """Test that LangfuseUsageDetails has the correct structure as defined in the commit"""
         # This test directly verifies the structure of the TypedDict
         # without relying on the LangFuseLogger class
-        
+
         # Create a dictionary that matches the LangfuseUsageDetails structure
         usage_details = {
             "input": 15,
@@ -200,19 +198,15 @@ class TestLangfuseUsageDetails(unittest.TestCase):
             "cache_creation_input_tokens": 7,
             "cache_read_input_tokens": 4
         }
-        
+
         # Verify the structure matches what we expect
         self.assertIn("input", usage_details)
         self.assertIn("output", usage_details)
         self.assertIn("cache_creation_input_tokens", usage_details)
         self.assertIn("cache_read_input_tokens", usage_details)
-        
+
         # Verify the values
         self.assertEqual(usage_details["input"], 15)
         self.assertEqual(usage_details["output"], 25)
         self.assertEqual(usage_details["cache_creation_input_tokens"], 7)
         self.assertEqual(usage_details["cache_read_input_tokens"], 4)
-
-
-if __name__ == "__main__":
-    unittest.main()

From 6d7d5ed07805fe40ec3ebdba2eca6fd769de7cdc Mon Sep 17 00:00:00 2001
From: Fabricio Ceschin <fceschin@todyl.com>
Date: Mon, 8 Sep 2025 11:37:24 -0400
Subject: [PATCH 08/19] Updating poetry.lock

---
 poetry.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/poetry.lock b/poetry.lock
index 771bb6d486e..41182d12bff 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -9182,4 +9182,4 @@ utils = ["numpydoc"]
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.8.1,<4.0, !=3.9.7"
-content-hash = "8ebdb444c0ff253857f90d3ec9926a58d23d4482c644fd158a047f0c0f892bb9"
+content-hash = "54374b906931afe92861b6b9b226751579c0d342f650bc8f7ebaf4c5882d6249"

From e437273ad52e2f1bd93a6d22574f9833e38839b0 Mon Sep 17 00:00:00 2001
From: Franklin <5235904+mrFranklin@users.noreply.github.com>
Date: Thu, 25 Sep 2025 00:57:21 +0800
Subject: [PATCH 09/19] doc: make the README document clearer (#14860)

* Update README.md

* Update README.md
---
 README.md | 33 +++++++++++++--------------------
 1 file changed, 13 insertions(+), 20 deletions(-)

diff --git a/README.md b/README.md
index f74889fbb27..0918d2b1fa4 100644
--- a/README.md
+++ b/README.md
@@ -350,13 +350,21 @@ curl 'http://0.0.0.0:4000/key/generate' \
 
 [**Read the Docs**](https://docs.litellm.ai/docs/)
 
-## Contributing
+## Run in Developer mode
+### Services
+1. Setup .env file in root
+2. Run dependant services `docker-compose up db prometheus`
 
-Interested in contributing? Contributions to LiteLLM Python SDK, Proxy Server, and LLM integrations are both accepted and highly encouraged!
+### Backend
+1. (In root) create virtual environment `python -m venv .venv`
+2. Activate virtual environment `source .venv/bin/activate`
+3. Install dependencies `pip install -e ".[all]"`
+4. Start proxy backend `python litellm/proxy_cli.py`
 
-**Quick start:** `git clone` → `make install-dev` → `make format` → `make lint` → `make test-unit`
-
-See our comprehensive [Contributing Guide (CONTRIBUTING.md)](CONTRIBUTING.md) for detailed instructions.
+### Frontend
+1. Navigate to `ui/litellm-dashboard`
+2. Install dependencies `npm install`
+3. Run `npm run dev` to start the dashboard
 
 # Enterprise
 For companies that need better security, user management and professional support
@@ -434,18 +442,3 @@ All these checks must pass before your PR can be merged.
 </a>
 
 
-## Run in Developer mode
-### Services
-1. Setup .env file in root
-2. Run dependant services `docker-compose up db prometheus`
-
-### Backend
-1. (In root) create virtual environment `python -m venv .venv`
-2. Activate virtual environment `source .venv/bin/activate`
-3. Install dependencies `pip install -e ".[all]"`
-4. Start proxy backend `python3 /path/to/litellm/proxy_cli.py`
-
-### Frontend
-1. Navigate to `ui/litellm-dashboard`
-2. Install dependencies `npm install`
-3. Run `npm run dev` to start the dashboard

From 22eef373eba848b02969f25d5c49b555ade8b57e Mon Sep 17 00:00:00 2001
From: onlylonly <onlylonly88@gmail.com>
Date: Thu, 25 Sep 2025 00:58:23 +0800
Subject: [PATCH 10/19] feat: New model - Add support for Qwen models family &
 Deepseek 3.1 to Amazon Bedrock (#14845)

* New model - Add Bedrock deepseek v3.1 model - "deepseek.v3-v1:0"

* New model - Add Bedrock Qwen models - "qwen.qwen3-coder-480b-a35b-v1:0", "qwen.qwen3-235b-a22b-2507-v1:0", "qwen.qwen3-coder-30b-a3b-v1:0", "qwen.qwen3-32b-v1:0"

* fix: add "deepseek.v3-v1:0" in  litellm/model_prices_and_context_window_backup.json
---
 litellm/constants.py                          |  5 ++
 ...odel_prices_and_context_window_backup.json | 60 +++++++++++++++++++
 model_prices_and_context_window.json          | 60 +++++++++++++++++++
 3 files changed, 125 insertions(+)

diff --git a/litellm/constants.py b/litellm/constants.py
index 005eb2bb6d0..6e70ae0671f 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -812,6 +812,11 @@ BEDROCK_EMBEDDING_PROVIDERS_LITERAL = Literal[
 ]
 
 BEDROCK_CONVERSE_MODELS = [
+    "qwen.qwen3-coder-480b-a35b-v1:0",
+    "qwen.qwen3-235b-a22b-2507-v1:0",
+    "qwen.qwen3-coder-30b-a3b-v1:0",
+    "qwen.qwen3-32b-v1:0",
+    "deepseek.v3-v1:0",
     "openai.gpt-oss-20b-1:0",
     "openai.gpt-oss-120b-1:0",
     "anthropic.claude-opus-4-1-20250805-v1:0",
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 832ff36bfb8..b0261d1bd43 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -7203,6 +7203,18 @@
         "supports_prompt_caching": true,
         "supports_tool_choice": true
     },
+    "deepseek.v3-v1:0": {
+        "input_cost_per_token": 5.8e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 163840,
+        "max_output_tokens": 81920,
+        "max_tokens": 163840,
+        "mode": "chat",
+        "output_cost_per_token": 1.68e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },     
     "dolphin": {
         "input_cost_per_token": 5e-07,
         "litellm_provider": "nlp_cloud",
@@ -17520,6 +17532,54 @@
         "mode": "chat",
         "output_cost_per_token": 2.8e-07
     },
+    "qwen.qwen3-coder-480b-a35b-v1:0": {
+        "input_cost_per_token": 2.2e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 262000,
+        "max_output_tokens": 65536,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 1.8e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
+    "qwen.qwen3-235b-a22b-2507-v1:0": {
+        "input_cost_per_token": 2.2e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 131072,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 8.8e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
+    "qwen.qwen3-coder-30b-a3b-v1:0": {
+        "input_cost_per_token": 1.5e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 131072,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 6.0e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    }, 
+    "qwen.qwen3-32b-v1:0": {
+        "input_cost_per_token": 1.5e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 16384,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 6.0e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },      
     "recraft/recraftv2": {
         "litellm_provider": "recraft",
         "mode": "image_generation",
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 832ff36bfb8..37be4e283aa 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -7203,6 +7203,18 @@
         "supports_prompt_caching": true,
         "supports_tool_choice": true
     },
+    "deepseek.v3-v1:0": {
+        "input_cost_per_token": 5.8e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 163840,
+        "max_output_tokens": 81920,
+        "max_tokens": 163840,
+        "mode": "chat",
+        "output_cost_per_token": 1.68e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },    
     "dolphin": {
         "input_cost_per_token": 5e-07,
         "litellm_provider": "nlp_cloud",
@@ -17520,6 +17532,54 @@
         "mode": "chat",
         "output_cost_per_token": 2.8e-07
     },
+    "qwen.qwen3-coder-480b-a35b-v1:0": {
+        "input_cost_per_token": 2.2e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 262000,
+        "max_output_tokens": 65536,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 1.8e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
+    "qwen.qwen3-235b-a22b-2507-v1:0": {
+        "input_cost_per_token": 2.2e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 131072,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 8.8e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
+    "qwen.qwen3-coder-30b-a3b-v1:0": {
+        "input_cost_per_token": 1.5e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 131072,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 6.0e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    }, 
+    "qwen.qwen3-32b-v1:0": {
+        "input_cost_per_token": 1.5e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 16384,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 6.0e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },           
     "recraft/recraftv2": {
         "litellm_provider": "recraft",
         "mode": "image_generation",

From 0cd91a82d28aebd63a5d62f68204c7877347fdf9 Mon Sep 17 00:00:00 2001
From: Mubashir Osmani <ilikewafflesomcuh@gmail.com>
Date: Wed, 24 Sep 2025 13:40:00 -0400
Subject: [PATCH 11/19] Added vertex_ai/qwen models and azure/gpt-5-codex
 (#14844)

* added qwen models and gpt-5-codex

* fix flaky test

* fix failing test
---
 model_prices_and_context_window.json          | 145 ++++++++++++++++++
 .../test_bedrock_completion.py                |  15 +-
 .../test_anthropic_passthrough.py             |   5 +-
 3 files changed, 159 insertions(+), 6 deletions(-)

diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 37be4e283aa..de1f0c5724e 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -2032,6 +2032,36 @@
         "supports_tool_choice": false,
         "supports_vision": true
     },
+    "azure/gpt-5-codex": {
+        "cache_read_input_token_cost": 1.25e-07,
+        "input_cost_per_token": 1.25e-06,
+        "litellm_provider": "azure",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "responses",
+        "output_cost_per_token": 1e-05,
+        "supported_endpoints": [
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "azure/gpt-5-mini": {
         "cache_read_input_token_cost": 2.5e-08,
         "input_cost_per_token": 2.5e-07,
@@ -5282,6 +5312,49 @@
         "supports_tool_choice": true,
         "supports_vision": true
     },
+    "deepseek-chat": {
+        "cache_read_input_token_cost": 6e-08,
+        "input_cost_per_token": 6e-07,
+        "litellm_provider": "deepseek",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 8192,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 1.7e-06,
+        "source": "https://api-docs.deepseek.com/quick_start/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true
+    },
+    "deepseek-reasoner": {
+        "cache_read_input_token_cost": 6e-08,
+        "input_cost_per_token": 6e-07,
+        "litellm_provider": "deepseek",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 65536,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 1.7e-06,
+        "source": "https://api-docs.deepseek.com/quick_start/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_function_calling": false,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": false,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": false
+    },
     "dashscope/qwen-coder": {
         "input_cost_per_token": 3e-07,
         "litellm_provider": "dashscope",
@@ -12427,6 +12500,36 @@
         "supports_tool_choice": false,
         "supports_vision": true
     },
+    "gpt-5-codex": {
+        "cache_read_input_token_cost": 1.25e-07,
+        "input_cost_per_token": 1.25e-06,
+        "litellm_provider": "openai",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "responses",
+        "output_cost_per_token": 1e-05,
+        "supported_endpoints": [
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": false,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": false,
+        "supports_prompt_caching": true,
+        "supports_reasoning": false,
+        "supports_response_schema": true,
+        "supports_system_messages": false,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "gpt-5-mini": {
         "cache_read_input_token_cost": 2.5e-08,
         "cache_read_input_token_cost_flex": 1.25e-08,
@@ -20527,6 +20630,24 @@
         "supports_function_calling": true,
         "supports_tool_choice": true
     },
+    "vertex_ai/deepseek-ai/deepseek-v3.1-maas": {
+        "input_cost_per_token": 1.35e-06,
+        "litellm_provider": "vertex_ai-deepseek_models",
+        "max_input_tokens": 163840,
+        "max_output_tokens": 32768,
+        "max_tokens": 163840,
+        "mode": "chat",
+        "output_cost_per_token": 5.4e-06,
+        "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
+        "supported_regions": [
+            "us-west2"
+        ],
+        "supports_assistant_prefill": true,
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
     "vertex_ai/deepseek-ai/deepseek-r1-0528-maas": {
         "input_cost_per_token": 1.35e-06,
         "litellm_provider": "vertex_ai-deepseek_models",
@@ -20940,6 +21061,30 @@
         "supports_function_calling": true,
         "supports_tool_choice": true
     },
+    "vertex_ai/qwen/qwen3-next-80b-a3b-instruct-maas": {
+        "input_cost_per_token": 1.5e-07,
+        "litellm_provider": "vertex_ai-qwen_models",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    },
+    "vertex_ai/qwen/qwen3-next-80b-a3b-thinking-maas": {
+        "input_cost_per_token": 1.5e-07,
+        "litellm_provider": "vertex_ai-qwen_models",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    },
     "vertex_ai/veo-2.0-generate-001": {
         "litellm_provider": "vertex_ai-video-models",
         "max_input_tokens": 1024,
diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py
index 91dff9f8636..d41448727d5 100644
--- a/tests/llm_translation/test_bedrock_completion.py
+++ b/tests/llm_translation/test_bedrock_completion.py
@@ -27,6 +27,7 @@ import litellm
 from litellm import (
     ModelResponse,
     RateLimitError,
+    ServiceUnavailableError,
     Timeout,
     completion,
     completion_cost,
@@ -2020,10 +2021,16 @@ def test_bedrock_context_window_error():
 
 def test_bedrock_converse_route():
     litellm.set_verbose = True
-    litellm.completion(
-        model="bedrock/converse/us.amazon.nova-pro-v1:0",
-        messages=[{"role": "user", "content": "Hello, world!"}],
-    )
+    try:
+        litellm.completion(
+            model="bedrock/converse/us.amazon.nova-pro-v1:0",
+            messages=[{"role": "user", "content": "Hello, world!"}],
+        )
+    except ServiceUnavailableError as e:
+        if "Too many requests" in str(e):
+            pytest.skip("Skipping test due to AWS Bedrock rate limiting")
+        else:
+            raise
 
 
 def test_bedrock_mapped_converse_models():
diff --git a/tests/pass_through_tests/test_anthropic_passthrough.py b/tests/pass_through_tests/test_anthropic_passthrough.py
index e0549d17fa2..002fb20e9e8 100644
--- a/tests/pass_through_tests/test_anthropic_passthrough.py
+++ b/tests/pass_through_tests/test_anthropic_passthrough.py
@@ -43,8 +43,9 @@ async def test_anthropic_basic_completion_with_headers():
                 json.dumps(response_json, indent=4, default=str),
             )
             reported_usage = response_json.get("usage", None)
-            anthropic_api_input_tokens = reported_usage.get("input_tokens", None)
-            anthropic_api_output_tokens = reported_usage.get("output_tokens", None)
+            # fix null checks for reported_usage
+            anthropic_api_input_tokens = reported_usage.get("input_tokens", None) if reported_usage else None
+            anthropic_api_output_tokens = reported_usage.get("output_tokens", None) if reported_usage else None
             litellm_call_id = response_headers.get("x-litellm-call-id")
 
             print(f"LiteLLM Call ID: {litellm_call_id}")

From c6cb36186c60fe70f65d0f7ef50c20622cee008c Mon Sep 17 00:00:00 2001
From: Luis Felipe Salazar Ucros <40307832+luisfucros@users.noreply.github.com>
Date: Wed, 24 Sep 2025 12:54:22 -0500
Subject: [PATCH 12/19] Add sambanova deepseek v3.1 and gpt-oss-120b (#14866)

* add sambanova deepseek v3.1 and gpt-oss-120b

* add sambanova deepseek v3.1 and gpt-oss-120b
---
 ...odel_prices_and_context_window_backup.json | 26 +++++++++++++++++++
 model_prices_and_context_window.json          | 26 +++++++++++++++++++
 2 files changed, 52 insertions(+)

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index b0261d1bd43..6d28cf1235d 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -18006,6 +18006,32 @@
         "supports_reasoning": true,
         "supports_tool_choice": true
     },
+    "sambanova/DeepSeek-V3.1": {
+        "max_tokens": 32768,
+        "max_input_tokens": 32768,
+        "max_output_tokens": 32768,
+        "input_cost_per_token": 3e-06,
+        "output_cost_per_token": 4.5e-06,
+        "litellm_provider": "sambanova",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_reasoning": true,
+        "source": "https://cloud.sambanova.ai/plans/pricing"
+    },
+    "sambanova/gpt-oss-120b": {
+        "max_tokens": 131072,
+        "max_input_tokens": 131072,
+        "max_output_tokens": 131072,
+        "input_cost_per_token": 3e-06,
+        "output_cost_per_token": 4.5e-06,
+        "litellm_provider": "sambanova",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_reasoning": true,
+        "source": "https://cloud.sambanova.ai/plans/pricing"
+    },
     "sample_spec": {
         "code_interpreter_cost_per_session": 0.0,
         "computer_use_input_cost_per_1k_tokens": 0.0,
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index de1f0c5724e..3742693ce67 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -18109,6 +18109,32 @@
         "supports_reasoning": true,
         "supports_tool_choice": true
     },
+    "sambanova/DeepSeek-V3.1": {
+        "max_tokens": 32768,
+        "max_input_tokens": 32768,
+        "max_output_tokens": 32768,
+        "input_cost_per_token": 3e-06,
+        "output_cost_per_token": 4.5e-06,
+        "litellm_provider": "sambanova",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_reasoning": true,
+        "source": "https://cloud.sambanova.ai/plans/pricing"
+    },
+    "sambanova/gpt-oss-120b": {
+        "max_tokens": 131072,
+        "max_input_tokens": 131072,
+        "max_output_tokens": 131072,
+        "input_cost_per_token": 3e-06,
+        "output_cost_per_token": 4.5e-06,
+        "litellm_provider": "sambanova",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_reasoning": true,
+        "source": "https://cloud.sambanova.ai/plans/pricing"
+    },
     "sample_spec": {
         "code_interpreter_cost_per_session": 0.0,
         "computer_use_input_cost_per_1k_tokens": 0.0,

From c69bac991be9f8690cd35d11d9bb95c020a37bb3 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Wed, 24 Sep 2025 10:55:56 -0700
Subject: [PATCH 13/19] [Fix] LakeraAI v2 Guardrail - Ensure exception is
 raised correctly  (#14867)

* fix exception lakera

* test lakera ai v2

* ruff fix
---
 .../guardrail_hooks/lakera_ai_v2.py           |  47 +++--
 litellm/proxy/proxy_config.yaml               |  13 ++
 tests/guardrails_tests/test_lakera_v2.py      | 179 +++++++++++++++++-
 3 files changed, 226 insertions(+), 13 deletions(-)

diff --git a/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py b/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py
index e167e73ac4d..b65664e00bd 100644
--- a/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py
+++ b/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py
@@ -3,9 +3,10 @@ import os
 from datetime import datetime
 from typing import Dict, List, Literal, Optional, Tuple, Union
 
+from fastapi import HTTPException
+
 import litellm
 from litellm._logging import verbose_proxy_logger
-from litellm.exceptions import GuardrailRaisedException
 from litellm.integrations.custom_guardrail import CustomGuardrail
 from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
@@ -237,9 +238,8 @@ class LakeraAIGuardrail(CustomGuardrail):
                 )
             else:
                 # If there are other violations or not set to mask PII, raise exception
-                raise GuardrailRaisedException(
-                    guardrail_name=self.guardrail_name,
-                    message="Lakera AI flagged this request. Please review the request and try again.",
+                raise self._get_http_exception_for_blocked_guardrail(
+                    lakera_guardrail_response
                 )
 
         #########################################################
@@ -304,9 +304,8 @@ class LakeraAIGuardrail(CustomGuardrail):
                 )
             else:
                 # If there are other violations or not set to mask PII, raise exception
-                raise GuardrailRaisedException(
-                    guardrail_name=self.guardrail_name,
-                    message="Lakera AI flagged this request. Please review the request and try again.",
+                raise self._get_http_exception_for_blocked_guardrail(
+                    lakera_guardrail_response
                 )
 
         #########################################################
@@ -327,8 +326,32 @@ class LakeraAIGuardrail(CustomGuardrail):
         if not lakera_response:
             return False
 
-        for item in lakera_response.get("payload", []) or []:
-            detector_type = item.get("detector_type", "") or ""
-            if not detector_type.startswith("pii/"):
-                return False
-        return True
+        # Check breakdown field for detected violations
+        breakdown = lakera_response.get("breakdown", []) or []
+        if not breakdown:
+            return False
+        
+        has_violations = False
+        for item in breakdown:
+            if item.get("detected", False):
+                has_violations = True
+                detector_type = item.get("detector_type", "") or ""
+                if not detector_type.startswith("pii/"):
+                    return False
+        
+        # Return True only if there are violations and they are all PII
+        return has_violations
+
+    def _get_http_exception_for_blocked_guardrail(
+        self, lakera_response: Optional[LakeraAIResponse]
+    ) -> HTTPException:
+        """
+        Get the HTTP exception for a blocked guardrail, similar to Bedrock's implementation.
+        """
+        return HTTPException(
+            status_code=400,
+            detail={
+                "error": "Violated guardrail policy",
+                "lakera_guardrail_response": lakera_response,
+            },
+        )
diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml
index addf2443d36..921b564407f 100644
--- a/litellm/proxy/proxy_config.yaml
+++ b/litellm/proxy/proxy_config.yaml
@@ -23,3 +23,16 @@ model_list:
     litellm_params:
       model: gemini/*
       api_key: os.environ/GEMINI_API_KEY
+
+
+guardrails:
+  - guardrail_name: lakera
+    litellm_params:
+      guardrail: lakera_v2
+      mode: pre_call
+      api_key: os.environ/LAKERA_API_KEY
+      default_on: false
+      project_id: project-9770817088
+      breakdown: true
+      payload: true
+      dev_info: true
diff --git a/tests/guardrails_tests/test_lakera_v2.py b/tests/guardrails_tests/test_lakera_v2.py
index 19c4424bee6..f3b2795a275 100644
--- a/tests/guardrails_tests/test_lakera_v2.py
+++ b/tests/guardrails_tests/test_lakera_v2.py
@@ -11,7 +11,8 @@ from litellm.proxy.guardrails.guardrail_hooks.lakera_ai_v2 import LakeraAIGuardr
 from litellm.types.guardrails import PiiEntityType, PiiAction
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.caching.caching import DualCache
-from litellm.exceptions import BlockedPiiEntityError
+from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
+from fastapi import HTTPException
 from litellm.types.utils import CallTypes as LitellmCallTypes
 
 
@@ -54,3 +55,179 @@ async def test_lakera_pre_call_hook_for_pii_masking():
     assert "4111-1111-1111-1111" not in user_message
     assert "test@example.com" not in user_message
 
+
+@pytest.mark.asyncio
+async def test_lakera_blocks_non_pii_violations():
+    """Test that Lakera guardrail blocks requests with non-PII violations like hate speech, violence, etc."""
+    
+    lakera_guardrail = LakeraAIGuardrail(
+        api_key="test_key",
+    )
+    
+    # Mock the call_v2_guard method to return a response similar to the user's example
+    mock_response = {
+        'payload': [],
+        'flagged': True,
+        'dev_info': {'git_revision': 'f0bc093a', 'git_timestamp': '2025-09-23T15:28:06+00:00', 'model_version': 'lakera-guard-1', 'version': '2.0.281'},
+        'metadata': {'request_uuid': 'b7cd4c8a-28aa-4285-a245-2befee514dbf'},
+        'breakdown': [
+            {'project_id': 'project-9770817088', 'policy_id': 'policy-lakera-default', 'detector_id': 'detector-lakera-default-moderated-content', 'detector_type': 'moderated_content/crime', 'detected': True, 'message_id': 0},
+            {'project_id': 'project-9770817088', 'policy_id': 'policy-lakera-default', 'detector_id': 'detector-lakera-default-moderated-content', 'detector_type': 'moderated_content/hate', 'detected': True, 'message_id': 0},
+            {'project_id': 'project-9770817088', 'policy_id': 'policy-lakera-default', 'detector_id': 'detector-lakera-default-moderated-content', 'detector_type': 'moderated_content/violence', 'detected': True, 'message_id': 0},
+            {'project_id': 'project-9770817088', 'policy_id': 'policy-lakera-default', 'detector_id': 'detector-lakera-default-prompt-attack', 'detector_type': 'prompt_attack', 'detected': True, 'message_id': 0},
+            {'project_id': 'project-9770817088', 'policy_id': 'policy-lakera-default', 'detector_id': 'detector-lakera-default-pii', 'detector_type': 'pii/email', 'detected': False, 'message_id': 0},
+        ]
+    }
+    
+    with patch.object(lakera_guardrail, 'call_v2_guard', new_callable=AsyncMock) as mock_call:
+        mock_call.return_value = (mock_response, {})
+        
+        # Create a sample request that would trigger violations
+        data = {
+            "messages": [
+                {"role": "user", "content": "Some harmful content that triggers violations"}
+            ],
+            "model": "gpt-3.5-turbo",
+            "metadata": {}
+        }
+        
+        # Mock objects needed for the pre-call hook
+        user_api_key_dict = UserAPIKeyAuth(api_key="test_key")
+        cache = DualCache()
+        
+        # The guardrail should raise an HTTPException for non-PII violations
+        with pytest.raises(HTTPException) as exc_info:
+            await lakera_guardrail.async_pre_call_hook(
+                user_api_key_dict=user_api_key_dict,
+                cache=cache,
+                data=data,
+                call_type="completion"
+            )
+        
+        # Verify the exception details include the Lakera response
+        assert exc_info.value.status_code == 400
+        assert "Violated guardrail policy" in str(exc_info.value.detail)
+        assert "lakera_guardrail_response" in exc_info.value.detail
+
+
+@pytest.mark.asyncio
+async def test_lakera_only_pii_violations_are_masked():
+    """Test that Lakera guardrail only masks PII violations and doesn't block the request."""
+    
+    lakera_guardrail = LakeraAIGuardrail(
+        api_key="test_key",
+    )
+    
+    # Mock response with only PII violations
+    mock_response = {
+        'payload': [
+            {'detector_type': 'pii/email', 'start': 10, 'end': 25, 'message_id': 0}
+        ],
+        'flagged': True,
+        'breakdown': [
+            {'project_id': 'project-9770817088', 'detector_type': 'pii/email', 'detected': True, 'message_id': 0},
+            {'project_id': 'project-9770817088', 'detector_type': 'moderated_content/hate', 'detected': False, 'message_id': 0},
+            {'project_id': 'project-9770817088', 'detector_type': 'prompt_attack', 'detected': False, 'message_id': 0},
+        ]
+    }
+    
+    with patch.object(lakera_guardrail, 'call_v2_guard', new_callable=AsyncMock) as mock_call:
+        mock_call.return_value = (mock_response, {})
+        
+        data = {
+            "messages": [
+                {"role": "user", "content": "My email test@example.com here"}
+            ],
+            "model": "gpt-3.5-turbo",
+            "metadata": {}
+        }
+        
+        user_api_key_dict = UserAPIKeyAuth(api_key="test_key")
+        cache = DualCache()
+        
+        # Should not raise an exception, just mask the PII
+        result = await lakera_guardrail.async_pre_call_hook(
+            user_api_key_dict=user_api_key_dict,
+            cache=cache,
+            data=data,
+            call_type="completion"
+        )
+        
+        # Verify the request was not blocked
+        assert result is not None
+        assert "messages" in result
+
+
+@pytest.mark.asyncio
+async def test_lakera_blocks_flagged_content_with_user_scenario():
+    """
+    Test the exact user scenario where Lakera flagged content but request went through.
+    This should now be blocked with the fix to check breakdown field instead of payload.
+    """
+    
+    lakera_guardrail = LakeraAIGuardrail(
+        api_key="test_key",
+    )
+    
+    # Mock response matching the exact user scenario
+    mock_response = {
+        'payload': [],  # Empty payload like in user's case
+        'flagged': True,
+        'dev_info': {'git_revision': 'f0bc093a', 'git_timestamp': '2025-09-23T15:28:06+00:00', 'model_version': 'lakera-guard-1', 'version': '2.0.281'},
+        'metadata': {'request_uuid': 'b7cd4c8a-28aa-4285-a245-2befee514dbf'},
+        'breakdown': [
+            {'project_id': 'project-9770817088', 'policy_id': 'policy-lakera-default', 'detector_id': 'detector-lakera-default-moderated-content', 'detector_type': 'moderated_content/crime', 'detected': True, 'message_id': 0},
+            {'project_id': 'project-9770817088', 'policy_id': 'policy-lakera-default', 'detector_id': 'detector-lakera-default-moderated-content', 'detector_type': 'moderated_content/hate', 'detected': True, 'message_id': 0},
+            {'project_id': 'project-9770817088', 'policy_id': 'policy-lakera-default', 'detector_id': 'detector-lakera-default-moderated-content', 'detector_type': 'moderated_content/profanity', 'detected': False, 'message_id': 0},
+            {'project_id': 'project-9770817088', 'policy_id': 'policy-lakera-default', 'detector_id': 'detector-lakera-default-moderated-content', 'detector_type': 'moderated_content/sexual', 'detected': False, 'message_id': 0},
+            {'project_id': 'project-9770817088', 'policy_id': 'policy-lakera-default', 'detector_id': 'detector-lakera-default-moderated-content', 'detector_type': 'moderated_content/violence', 'detected': True, 'message_id': 0},
+            {'project_id': 'project-9770817088', 'policy_id': 'policy-lakera-default', 'detector_id': 'detector-lakera-default-moderated-content', 'detector_type': 'moderated_content/weapons', 'detected': True, 'message_id': 0},
+            {'project_id': 'project-9770817088', 'policy_id': 'policy-lakera-default', 'detector_id': 'detector-lakera-default-pii', 'detector_type': 'pii/address', 'detected': False, 'message_id': 0},
+            {'project_id': 'project-9770817088', 'policy_id': 'policy-lakera-default', 'detector_id': 'detector-lakera-default-pii', 'detector_type': 'pii/credit_card', 'detected': False, 'message_id': 0},
+            {'project_id': 'project-9770817088', 'policy_id': 'policy-lakera-default', 'detector_id': 'detector-lakera-default-pii', 'detector_type': 'pii/email', 'detected': False, 'message_id': 0},
+            {'project_id': 'project-9770817088', 'policy_id': 'policy-lakera-default', 'detector_id': 'detector-lakera-default-pii', 'detector_type': 'pii/iban_code', 'detected': False, 'message_id': 0},
+            {'project_id': 'project-9770817088', 'policy_id': 'policy-lakera-default', 'detector_id': 'detector-lakera-default-pii', 'detector_type': 'pii/ip_address', 'detected': False, 'message_id': 0},
+            {'project_id': 'project-9770817088', 'policy_id': 'policy-lakera-default', 'detector_id': 'detector-lakera-default-pii', 'detector_type': 'pii/name', 'detected': False, 'message_id': 0},
+            {'project_id': 'project-9770817088', 'policy_id': 'policy-lakera-default', 'detector_id': 'detector-lakera-default-pii', 'detector_type': 'pii/phone_number', 'detected': False, 'message_id': 0},
+            {'project_id': 'project-9770817088', 'policy_id': 'policy-lakera-default', 'detector_id': 'detector-lakera-default-pii', 'detector_type': 'pii/us_social_security_number', 'detected': False, 'message_id': 0},
+            {'project_id': 'project-9770817088', 'policy_id': 'policy-lakera-default', 'detector_id': 'detector-lakera-default-prompt-attack', 'detector_type': 'prompt_attack', 'detected': True, 'message_id': 0},
+            {'project_id': 'project-9770817088', 'policy_id': 'policy-lakera-default', 'detector_id': 'detector-lakera-default-unknown-links', 'detector_type': 'unknown_links', 'detected': False, 'message_id': 0}
+        ]
+    }
+    
+    with patch.object(lakera_guardrail, 'call_v2_guard', new_callable=AsyncMock) as mock_call:
+        mock_call.return_value = (mock_response, {})
+        
+        # Create a sample request that would trigger violations
+        data = {
+            "messages": [
+                {"role": "user", "content": "Some harmful content that should be blocked"}
+            ],
+            "model": "gpt-3.5-turbo",
+            "metadata": {}
+        }
+        
+        # Mock objects needed for the pre-call hook
+        user_api_key_dict = UserAPIKeyAuth(api_key="test_key")
+        cache = DualCache()
+        
+        # With the fix, this should now raise an HTTPException instead of letting the request through
+        with pytest.raises(HTTPException) as exc_info:
+            await lakera_guardrail.async_pre_call_hook(
+                user_api_key_dict=user_api_key_dict,
+                cache=cache,
+                data=data,
+                call_type="completion"
+            )
+        
+        # Verify the exception details
+        assert exc_info.value.status_code == 400
+        assert "Violated guardrail policy" in str(exc_info.value.detail)
+        assert "lakera_guardrail_response" in exc_info.value.detail
+        
+        # Verify the full response is included in the exception
+        lakera_response = exc_info.value.detail["lakera_guardrail_response"]
+        assert lakera_response["flagged"] is True
+        assert lakera_response["metadata"]["request_uuid"] == "b7cd4c8a-28aa-4285-a245-2befee514dbf"
+        assert len(lakera_response["breakdown"]) == 16  # All the breakdown items from the user's scenario
+

From 3f5b4c84eb42228357c6ddaa67c349d77abe6787 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Wed, 24 Sep 2025 11:21:14 -0700
Subject: [PATCH 14/19] feat: Add /user/list route to LiteLLMRoutes (#14868)

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: ishaan <ishaan@berri.ai>
---
 litellm/proxy/_types.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 9d5298c30f8..a92b7d8ad2e 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -422,6 +422,7 @@ class LiteLLMRoutes(enum.Enum):
         "/user/update",
         "/user/delete",
         "/user/info",
+        "/user/list",
         # team
         "/team/new",
         "/team/update",

From 1816176f9d519e742878d5b3f809e913c971fee2 Mon Sep 17 00:00:00 2001
From: Copilot <198982749+Copilot@users.noreply.github.com>
Date: Wed, 24 Sep 2025 11:54:06 -0700
Subject: [PATCH 15/19] [Memory Leak Fix] Fix InMemoryCache unbounded growth
 when TTLs are set (#14869)

* Initial plan

* Fix InMemoryCache unbounded growth issue when TTLs are set

Co-authored-by: ishaan-jaff <29436595+ishaan-jaff@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: ishaan-jaff <29436595+ishaan-jaff@users.noreply.github.com>
---
 litellm/caching/in_memory_cache.py            | 24 ++++-
 .../caching/test_in_memory_cache.py           | 98 +++++++++++++++++++
 2 files changed, 118 insertions(+), 4 deletions(-)

diff --git a/litellm/caching/in_memory_cache.py b/litellm/caching/in_memory_cache.py
index 63869474d47..082cac791f2 100644
--- a/litellm/caching/in_memory_cache.py
+++ b/litellm/caching/in_memory_cache.py
@@ -36,7 +36,7 @@ class InMemoryCache(BaseCache):
         max_size_in_memory [int]: Maximum number of items in cache. done to prevent memory leaks. Use 200 items as a default
         """
         self.max_size_in_memory = (
-            max_size_in_memory or 200
+            max_size_in_memory if max_size_in_memory is not None else 200
         )  # set an upper bound of 200 items in-memory
         self.default_ttl = default_ttl or 600
         self.max_size_per_item = (
@@ -103,20 +103,32 @@ class InMemoryCache(BaseCache):
     def evict_cache(self):
         """
         Eviction policy:
-        - check if any items in ttl_dict are expired -> remove them from ttl_dict and cache_dict
+        1. First, remove expired items from ttl_dict and cache_dict
+        2. If cache is still at or above max_size_in_memory, evict items with earliest expiration times
 
 
         This guarantees the following:
-        - 1. When item ttl not set: At minimumm each item will remain in memory for 5 minutes
-        - 2. When ttl is set: the item will remain in memory for at least that amount of time
+        - 1. When item ttl not set: At minimum each item will remain in memory for the default ttl
+        - 2. When ttl is set: the item will remain in memory for at least that amount of time, unless cache size requires eviction
         - 3. the size of in-memory cache is bounded
 
         """
         current_time = time.time()
+        
+        # Step 1: Remove expired items
         expired_keys = [key for key, ttl in self.ttl_dict.items() if current_time > ttl]
         for key in expired_keys:
             self._remove_key(key)
 
+        # Step 2: If cache is still full, evict items with earliest expiration times
+        if len(self.cache_dict) >= self.max_size_in_memory:
+            # Sort by expiration time (earliest first) and evict until we're under the limit
+            items_by_expiration = sorted(self.ttl_dict.items(), key=lambda x: x[1])
+            keys_to_evict = items_by_expiration[:len(self.cache_dict) - self.max_size_in_memory + 1]
+            
+            for key, _ in keys_to_evict:
+                self._remove_key(key)
+
         # de-reference the removed item
         # https://www.geeksforgeeks.org/diagnosing-and-fixing-memory-leaks-in-python/
         # One of the most common causes of memory leaks in Python is the retention of objects that are no longer being used.
@@ -135,6 +147,10 @@ class InMemoryCache(BaseCache):
             return False
 
     def set_cache(self, key, value, **kwargs):
+        # Handle the edge case where max_size_in_memory is 0
+        if self.max_size_in_memory == 0:
+            return  # Don't cache anything if max size is 0
+            
         if len(self.cache_dict) >= self.max_size_in_memory:
             # only evict when cache is full
             self.evict_cache()
diff --git a/tests/test_litellm/caching/test_in_memory_cache.py b/tests/test_litellm/caching/test_in_memory_cache.py
index 72f264b8b7d..616c60c74a0 100644
--- a/tests/test_litellm/caching/test_in_memory_cache.py
+++ b/tests/test_litellm/caching/test_in_memory_cache.py
@@ -88,3 +88,101 @@ def test_in_memory_cache_ttl_allow_override():
     new_ttl_time = in_memory_cache.ttl_dict["new-fake-key"]
     assert new_ttl_time is not None
     assert new_ttl_time != initial_ttl_time
+
+
+def test_in_memory_cache_max_size_with_ttl():
+    """
+    Test that max_size_in_memory is respected even when all items have long TTLs.
+    This tests the fix for the unbounded growth issue.
+    """
+    in_memory_cache = InMemoryCache(max_size_in_memory=3)
+    long_ttl = 86400  # 1 day
+    
+    # Fill the cache to max capacity
+    for i in range(3):
+        in_memory_cache.set_cache(key=f"key_{i}", value=f"value_{i}", ttl=long_ttl)
+        time.sleep(0.01)  # Small delay to ensure different timestamps
+    
+    assert len(in_memory_cache.cache_dict) == 3
+    assert len(in_memory_cache.ttl_dict) == 3
+    
+    # Add another item - should evict the earliest item
+    in_memory_cache.set_cache(key="key_3", value="value_3", ttl=long_ttl)
+    
+    # Cache should still be at max size, not larger
+    assert len(in_memory_cache.cache_dict) == 3
+    assert len(in_memory_cache.ttl_dict) == 3
+    
+    # key_0 should have been evicted (it was added first)
+    assert "key_0" not in in_memory_cache.cache_dict
+    assert "key_0" not in in_memory_cache.ttl_dict
+    
+    # Other keys should still be present
+    assert "key_1" in in_memory_cache.cache_dict
+    assert "key_2" in in_memory_cache.cache_dict
+    assert "key_3" in in_memory_cache.cache_dict
+
+
+def test_in_memory_cache_expired_items_evicted_first():
+    """
+    Test that expired items are evicted before non-expired items when cache is full.
+    """
+    in_memory_cache = InMemoryCache(max_size_in_memory=3)
+    
+    # Add items with short TTL that will expire
+    in_memory_cache.set_cache(key="expired_1", value="value_1", ttl=1)
+    in_memory_cache.set_cache(key="expired_2", value="value_2", ttl=1)
+    
+    # Add item with long TTL
+    in_memory_cache.set_cache(key="long_lived", value="value_long", ttl=86400)
+    
+    assert len(in_memory_cache.cache_dict) == 3
+    
+    # Wait for short TTL items to expire
+    time.sleep(2)
+    
+    # Add new item - should evict expired items first, not the long-lived one
+    in_memory_cache.set_cache(key="new_item", value="new_value", ttl=86400)
+    
+    # Long-lived item should still be present
+    assert "long_lived" in in_memory_cache.cache_dict
+    assert "new_item" in in_memory_cache.cache_dict
+    
+    # Expired items should be gone
+    assert "expired_1" not in in_memory_cache.cache_dict
+    assert "expired_2" not in in_memory_cache.cache_dict
+    assert "expired_1" not in in_memory_cache.ttl_dict
+    assert "expired_2" not in in_memory_cache.ttl_dict
+
+
+def test_in_memory_cache_eviction_order():
+    """
+    Test that when non-expired items need to be evicted, those with earliest expiration times are evicted first.
+    """
+    in_memory_cache = InMemoryCache(max_size_in_memory=2)
+    
+    # Add items with different TTLs
+    now = time.time()
+    in_memory_cache.set_cache(key="early_expire", value="value_1", ttl=100)  # expires in 100 seconds
+    time.sleep(0.01)
+    in_memory_cache.set_cache(key="late_expire", value="value_2", ttl=200)   # expires in 200 seconds
+    
+    # Verify TTL order
+    early_ttl = in_memory_cache.ttl_dict["early_expire"]
+    late_ttl = in_memory_cache.ttl_dict["late_expire"]
+    assert early_ttl < late_ttl, "early_expire should have earlier expiration time"
+    
+    assert len(in_memory_cache.cache_dict) == 2
+    
+    # Add third item - should evict the one with earliest expiration time
+    in_memory_cache.set_cache(key="new_item", value="value_3", ttl=300)
+    
+    assert len(in_memory_cache.cache_dict) == 2
+    
+    # Item with earliest expiration should be evicted
+    assert "early_expire" not in in_memory_cache.cache_dict
+    assert "early_expire" not in in_memory_cache.ttl_dict
+    
+    # Items with later expiration should remain
+    assert "late_expire" in in_memory_cache.cache_dict
+    assert "new_item" in in_memory_cache.cache_dict

From 66a88d3761a3d9e3cef530a867be7bf3431a18b3 Mon Sep 17 00:00:00 2001
From: Ishaan Jaffer <ishaanjaffer0324@gmail.com>
Date: Wed, 24 Sep 2025 11:54:44 -0700
Subject: [PATCH 16/19] fix ci/cd

---
 .circleci/config.yml                          |   2 +-
 ...odel_prices_and_context_window_backup.json | 149 +++++++++++++++++-
 2 files changed, 148 insertions(+), 3 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 0e53cfc0edb..ef6445ca0ca 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -144,7 +144,7 @@ jobs:
           name: Linting Testing
           command: |
             cd litellm
-            pip install "cryptography<40.0.0"
+            pip install "cryptography>=43.0.1"
             python -m pip install types-requests types-setuptools types-redis types-PyYAML
             if ! python -m mypy . \
                 --config-file mypy.ini \
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 6d28cf1235d..3742693ce67 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -2032,6 +2032,36 @@
         "supports_tool_choice": false,
         "supports_vision": true
     },
+    "azure/gpt-5-codex": {
+        "cache_read_input_token_cost": 1.25e-07,
+        "input_cost_per_token": 1.25e-06,
+        "litellm_provider": "azure",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "responses",
+        "output_cost_per_token": 1e-05,
+        "supported_endpoints": [
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "azure/gpt-5-mini": {
         "cache_read_input_token_cost": 2.5e-08,
         "input_cost_per_token": 2.5e-07,
@@ -5282,6 +5312,49 @@
         "supports_tool_choice": true,
         "supports_vision": true
     },
+    "deepseek-chat": {
+        "cache_read_input_token_cost": 6e-08,
+        "input_cost_per_token": 6e-07,
+        "litellm_provider": "deepseek",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 8192,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 1.7e-06,
+        "source": "https://api-docs.deepseek.com/quick_start/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true
+    },
+    "deepseek-reasoner": {
+        "cache_read_input_token_cost": 6e-08,
+        "input_cost_per_token": 6e-07,
+        "litellm_provider": "deepseek",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 65536,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 1.7e-06,
+        "source": "https://api-docs.deepseek.com/quick_start/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_function_calling": false,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": false,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": false
+    },
     "dashscope/qwen-coder": {
         "input_cost_per_token": 3e-07,
         "litellm_provider": "dashscope",
@@ -7214,7 +7287,7 @@
         "supports_function_calling": true,
         "supports_reasoning": true,
         "supports_tool_choice": true
-    },     
+    },    
     "dolphin": {
         "input_cost_per_token": 5e-07,
         "litellm_provider": "nlp_cloud",
@@ -12427,6 +12500,36 @@
         "supports_tool_choice": false,
         "supports_vision": true
     },
+    "gpt-5-codex": {
+        "cache_read_input_token_cost": 1.25e-07,
+        "input_cost_per_token": 1.25e-06,
+        "litellm_provider": "openai",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "responses",
+        "output_cost_per_token": 1e-05,
+        "supported_endpoints": [
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": false,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": false,
+        "supports_prompt_caching": true,
+        "supports_reasoning": false,
+        "supports_response_schema": true,
+        "supports_system_messages": false,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "gpt-5-mini": {
         "cache_read_input_token_cost": 2.5e-08,
         "cache_read_input_token_cost_flex": 1.25e-08,
@@ -17579,7 +17682,7 @@
         "supports_function_calling": true,
         "supports_reasoning": true,
         "supports_tool_choice": true
-    },      
+    },           
     "recraft/recraftv2": {
         "litellm_provider": "recraft",
         "mode": "image_generation",
@@ -20553,6 +20656,24 @@
         "supports_function_calling": true,
         "supports_tool_choice": true
     },
+    "vertex_ai/deepseek-ai/deepseek-v3.1-maas": {
+        "input_cost_per_token": 1.35e-06,
+        "litellm_provider": "vertex_ai-deepseek_models",
+        "max_input_tokens": 163840,
+        "max_output_tokens": 32768,
+        "max_tokens": 163840,
+        "mode": "chat",
+        "output_cost_per_token": 5.4e-06,
+        "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
+        "supported_regions": [
+            "us-west2"
+        ],
+        "supports_assistant_prefill": true,
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
     "vertex_ai/deepseek-ai/deepseek-r1-0528-maas": {
         "input_cost_per_token": 1.35e-06,
         "litellm_provider": "vertex_ai-deepseek_models",
@@ -20966,6 +21087,30 @@
         "supports_function_calling": true,
         "supports_tool_choice": true
     },
+    "vertex_ai/qwen/qwen3-next-80b-a3b-instruct-maas": {
+        "input_cost_per_token": 1.5e-07,
+        "litellm_provider": "vertex_ai-qwen_models",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    },
+    "vertex_ai/qwen/qwen3-next-80b-a3b-thinking-maas": {
+        "input_cost_per_token": 1.5e-07,
+        "litellm_provider": "vertex_ai-qwen_models",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    },
     "vertex_ai/veo-2.0-generate-001": {
         "litellm_provider": "vertex_ai-video-models",
         "max_input_tokens": 1024,

From 268b66b7ea826c5b6be79ea090f9d0fa3626b216 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 24 Sep 2025 16:13:13 -0700
Subject: [PATCH 17/19] Bump tar-fs from 3.0.10 to 3.1.1 in /docs/my-website
 (#14872)

Bumps [tar-fs](https://github.com/mafintosh/tar-fs) from 3.0.10 to 3.1.1.
- [Commits](https://github.com/mafintosh/tar-fs/compare/v3.0.10...v3.1.1)

---
updated-dependencies:
- dependency-name: tar-fs
  dependency-version: 3.1.1
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 docs/my-website/package-lock.json | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/docs/my-website/package-lock.json b/docs/my-website/package-lock.json
index 4b37e2be11d..b71a15cc8e6 100644
--- a/docs/my-website/package-lock.json
+++ b/docs/my-website/package-lock.json
@@ -17120,9 +17120,10 @@
       }
     },
     "node_modules/prebuild-install/node_modules/tar-fs": {
-      "version": "2.1.3",
-      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.3.tgz",
-      "integrity": "sha512-090nwYJDmlhwFwEW3QQl+vaNnxsO2yVsd45eTKRBzSzu+hlb1w2K9inVq5b0ngXuLVqQ4ApvsUHHnu/zQNkWAg==",
+      "version": "2.1.4",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz",
+      "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==",
+      "license": "MIT",
       "dependencies": {
         "chownr": "^1.1.1",
         "mkdirp-classic": "^0.5.2",
@@ -19295,9 +19296,10 @@
       }
     },
     "node_modules/tar-fs": {
-      "version": "3.0.10",
-      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.10.tgz",
-      "integrity": "sha512-C1SwlQGNLe/jPNqapK8epDsXME7CAJR5RL3GcE6KWx1d9OUByzoHVcbu1VPI8tevg9H8Alae0AApHHFGzrD5zA==",
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.1.tgz",
+      "integrity": "sha512-LZA0oaPOc2fVo82Txf3gw+AkEd38szODlptMYejQUhndHMLQ9M059uXR+AfS7DNo0NpINvSqDsvyaCrBVkptWg==",
+      "license": "MIT",
       "dependencies": {
         "pump": "^3.0.0",
         "tar-stream": "^3.1.5"

From df88b0656e000f48738b745abc89dfec3b5f8380 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Wed, 24 Sep 2025 16:13:41 -0700
Subject: [PATCH 18/19] test: update .json

---
 .../langfuse_expected_request_body/completion.json       | 9 ++++++++-
 .../completion_with_bedrock_call.json                    | 8 +++++++-
 .../completion_with_complex_metadata.json                | 6 ++++++
 .../completion_with_langfuse_metadata.json               | 6 ++++++
 .../completion_with_no_choices.json                      | 6 ++++++
 .../completion_with_router.json                          | 6 ++++++
 .../completion_with_tags.json                            | 6 ++++++
 .../completion_with_tags_stream.json                     | 6 ++++++
 .../completion_with_vertex_call.json                     | 6 ++++++
 .../langfuse_expected_request_body/complex_metadata.json | 6 ++++++
 .../complex_metadata_2.json                              | 6 ++++++
 .../langfuse_expected_request_body/empty_metadata.json   | 6 ++++++
 .../metadata_with_function.json                          | 6 ++++++
 .../metadata_with_lock.json                              | 6 ++++++
 .../langfuse_expected_request_body/nested_metadata.json  | 6 ++++++
 .../langfuse_expected_request_body/simple_metadata.json  | 6 ++++++
 .../langfuse_expected_request_body/simple_metadata2.json | 6 ++++++
 .../langfuse_expected_request_body/simple_metadata3.json | 6 ++++++
 18 files changed, 111 insertions(+), 2 deletions(-)

diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion.json
index bede5753a2a..b995df2d445 100644
--- a/tests/logging_callback_tests/langfuse_expected_request_body/completion.json
+++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion.json
@@ -68,7 +68,14 @@
                     "output": 20,
                     "unit": "TOKENS",
                     "totalCost": 5.4999999999999995e-05
-                }
+                },
+                "usageDetails": {
+                        "cache_creation_input_tokens": 0,
+                        "cache_read_input_tokens": 0,
+                        "input": 10,
+                        "output": 20
+                    },
+                "traceId": "litellm-test-6a51ae70-a4e7-499e-afcd-dce2a3b31850"
             },
             "timestamp": "2025-01-16T19:28:55.125258Z"
         }
diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_bedrock_call.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_bedrock_call.json
index aa96e5949e5..d9f52477fc8 100644
--- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_bedrock_call.json
+++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_bedrock_call.json
@@ -63,7 +63,13 @@
                     "output": 10,
                     "unit": "TOKENS",
                     "totalCost": 0.00018
-                }
+                },
+                "usageDetails": {
+                        "cache_creation_input_tokens": 0,
+                        "cache_read_input_tokens": 0,
+                        "input": 10,
+                        "output": 10
+                    }
             },
             "timestamp": "2025-05-26T21:13:16.797156Z"
         }
diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json
index 1e60a0479b1..348fe5956da 100644
--- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json
+++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json
@@ -109,6 +109,12 @@
                     "output": 20,
                     "unit": "TOKENS",
                     "totalCost": 5.4999999999999995e-05
+                },
+                "usageDetails": {
+                    "cache_creation_input_tokens": 0,
+                    "cache_read_input_tokens": 0,
+                    "input": 10,
+                    "output": 20
                 }
             },
             "timestamp": "2025-01-22T17:27:51.703046Z"
diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json
index a217a901285..b63bedf16a2 100644
--- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json
+++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json
@@ -87,6 +87,12 @@
                     "output": 20,
                     "unit": "TOKENS",
                     "totalCost": 5.4999999999999995e-05
+                },
+                "usageDetails": {
+                    "cache_creation_input_tokens": 0,
+                    "cache_read_input_tokens": 0,
+                    "input": 10,
+                    "output": 20
                 }
             },
             "timestamp": "2025-01-22T17:19:11.235541Z"
diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json
index be21c297dcc..f4242d7edc3 100644
--- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json
+++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json
@@ -58,6 +58,12 @@
                     "output": 10,
                     "unit": "TOKENS",
                     "totalCost": 3.5e-05
+                },
+                "usageDetails": {
+                    "cache_creation_input_tokens": 0,
+                    "cache_read_input_tokens": 0,
+                    "input": 10,
+                    "output": 10
                 }
             },
             "timestamp": "2025-02-07T00:23:27.670175Z"
diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_router.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_router.json
index 021c2b1b73c..84ea9768f01 100644
--- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_router.json
+++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_router.json
@@ -73,6 +73,12 @@
                     "output": 10,
                     "unit": "TOKENS",
                     "totalCost": 3.5e-05
+                },
+                "usageDetails": {
+                    "cache_creation_input_tokens": 0,
+                    "cache_read_input_tokens": 0,
+                    "input": 10,
+                    "output": 10
                 }
             },
             "timestamp": "2025-05-24T17:01:19.408586Z"
diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json
index 07fca9daafb..e7442ce0a02 100644
--- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json
+++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json
@@ -77,6 +77,12 @@
                     "output": 20,
                     "unit": "TOKENS",
                     "totalCost": 5.4999999999999995e-05
+                },
+                "usageDetails": {
+                    "cache_creation_input_tokens": 0,
+                    "cache_read_input_tokens": 0,
+                    "input": 10,
+                    "output": 20
                 }
             },
             "timestamp": "2025-01-22T15:31:28.964179Z"
diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json
index d7003c3f99e..04c24c8963a 100644
--- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json
+++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json
@@ -77,6 +77,12 @@
                     "output": 20,
                     "unit": "TOKENS",
                     "totalCost": 5.4999999999999995e-05
+                },
+                "usageDetails": {
+                    "cache_creation_input_tokens": 0,
+                    "cache_read_input_tokens": 0,
+                    "input": 10,
+                    "output": 20
                 }
             },
             "timestamp": "2025-01-22T16:38:26.017252Z"
diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_vertex_call.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_vertex_call.json
index 51a4fac0579..3e27f5b54b4 100644
--- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_vertex_call.json
+++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_vertex_call.json
@@ -61,6 +61,12 @@
                     "output": 10,
                     "unit": "TOKENS",
                     "totalCost": 7.5e-06
+                },
+                "usageDetails": {
+                    "cache_creation_input_tokens": 0,
+                    "cache_read_input_tokens": 0,
+                    "input": 10,
+                    "output": 10
                 }
             },
             "timestamp": "2025-05-26T21:15:40.610953Z"
diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json
index e8f01f6d723..a21ab058fcd 100644
--- a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json
+++ b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json
@@ -84,6 +84,12 @@
                     "output": 20,
                     "unit": "TOKENS",
                     "totalCost": 5.4999999999999995e-05
+                },
+                "usageDetails": {
+                    "cache_creation_input_tokens": 0,
+                    "cache_read_input_tokens": 0,
+                    "input": 10,
+                    "output": 20
                 }
             },
             "timestamp": "2025-01-22T17:59:39.368310Z"
diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json
index 786fe20e7b4..3f400f65914 100644
--- a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json
+++ b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json
@@ -76,6 +76,12 @@
                     "output": 20,
                     "unit": "TOKENS",
                     "totalCost": 5.4999999999999995e-05
+                },
+                "usageDetails": {
+                    "cache_creation_input_tokens": 0,
+                    "cache_read_input_tokens": 0,
+                    "input": 10,
+                    "output": 20
                 }
             },
             "timestamp": "2025-01-22T18:06:50.959850Z"
diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json
index 69db4314f0f..1873d4a6ea1 100644
--- a/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json
+++ b/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json
@@ -70,6 +70,12 @@
                     "output": 20,
                     "unit": "TOKENS",
                     "totalCost": 5.4999999999999995e-05
+                },
+                "usageDetails": {
+                    "cache_creation_input_tokens": 0,
+                    "cache_read_input_tokens": 0,
+                    "input": 10,
+                    "output": 20
                 }
             },
             "timestamp": "2025-01-22T17:59:32.889548Z"
diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json
index eae3134555e..34e3c9f8daf 100644
--- a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json
+++ b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json
@@ -70,6 +70,12 @@
                     "output": 20,
                     "unit": "TOKENS",
                     "totalCost": 5.4999999999999995e-05
+                },
+                "usageDetails": {
+                    "cache_creation_input_tokens": 0,
+                    "cache_read_input_tokens": 0,
+                    "input": 10,
+                    "output": 20
                 }
             },
             "timestamp": "2025-01-22T17:59:36.162997Z"
diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json
index 69db4314f0f..1873d4a6ea1 100644
--- a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json
+++ b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json
@@ -70,6 +70,12 @@
                     "output": 20,
                     "unit": "TOKENS",
                     "totalCost": 5.4999999999999995e-05
+                },
+                "usageDetails": {
+                    "cache_creation_input_tokens": 0,
+                    "cache_read_input_tokens": 0,
+                    "input": 10,
+                    "output": 20
                 }
             },
             "timestamp": "2025-01-22T17:59:32.889548Z"
diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json
index c4fe594fa14..4d6ce12ec6d 100644
--- a/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json
+++ b/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json
@@ -76,6 +76,12 @@
                     "output": 20,
                     "unit": "TOKENS",
                     "totalCost": 5.4999999999999995e-05
+                },
+                "usageDetails": {
+                    "cache_creation_input_tokens": 0,
+                    "cache_read_input_tokens": 0,
+                    "input": 10,
+                    "output": 20
                 }
             },
             "timestamp": "2025-01-22T17:55:28.855732Z"
diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json
index d348ac50392..5e5edc795ec 100644
--- a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json
+++ b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json
@@ -76,6 +76,12 @@
                     "output": 20,
                     "unit": "TOKENS",
                     "totalCost": 5.4999999999999995e-05
+                },
+                "usageDetails": {
+                    "cache_creation_input_tokens": 0,
+                    "cache_read_input_tokens": 0,
+                    "input": 10,
+                    "output": 20
                 }
             },
             "timestamp": "2025-01-22T17:53:53.754511Z"
diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json
index d5468d302c9..7bbf4e4eeee 100644
--- a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json
+++ b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json
@@ -80,6 +80,12 @@
                     "output": 20,
                     "unit": "TOKENS",
                     "totalCost": 5.4999999999999995e-05
+                },
+                "usageDetails": {
+                    "cache_creation_input_tokens": 0,
+                    "cache_read_input_tokens": 0,
+                    "input": 10,
+                    "output": 20
                 }
             },
             "timestamp": "2025-01-22T17:56:35.478171Z"
diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json
index 1107766993a..fcdcb47aea5 100644
--- a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json
+++ b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json
@@ -84,6 +84,12 @@
                     "output": 20,
                     "unit": "TOKENS",
                     "totalCost": 5.4999999999999995e-05
+                },
+                "usageDetails": {
+                    "cache_creation_input_tokens": 0,
+                    "cache_read_input_tokens": 0,
+                    "input": 10,
+                    "output": 20
                 }
             },
             "timestamp": "2025-01-22T17:56:38.787196Z"

From e75d8b711e156d1528d560a53263288238cdba99 Mon Sep 17 00:00:00 2001
From: Alex Shoop <alexander.shoop@datarobot.com>
Date: Thu, 25 Sep 2025 09:09:30 +0900
Subject: [PATCH 19/19] Fix: make `pondpond` as optional dependency for `proxy`
 extras, disable object pooling gracefully (#14863)

* pondpond optional dep proxy extra

* lock
---
 litellm/litellm_core_utils/object_pooling.py  | 38 ++++++++++++++++---
 poetry.lock                                   | 10 +++--
 pyproject.toml                                |  3 +-
 .../test_object_pooling.py                    |  3 +-
 4 files changed, 43 insertions(+), 11 deletions(-)

diff --git a/litellm/litellm_core_utils/object_pooling.py b/litellm/litellm_core_utils/object_pooling.py
index 846e6536f80..81c3ec2e133 100644
--- a/litellm/litellm_core_utils/object_pooling.py
+++ b/litellm/litellm_core_utils/object_pooling.py
@@ -16,11 +16,35 @@ Memory Management Strategy:
 
 from typing import Any, Callable, Optional, Type, TypeVar
 
-from pond import Pond, PooledObject, PooledObjectFactory
+try:
+    from pond import Pond, PooledObject, PooledObjectFactory  # type: ignore
+    POND_AVAILABLE = True
+except ImportError:  # pragma: no cover
+    POND_AVAILABLE = False
+    class Pond:  # type: ignore
+        def __init__(self, *args: Any, **kwargs: Any) -> None:
+            pass
+
+        def register(self, *args: Any, **kwargs: Any) -> None:
+            pass
+
+        def borrow(self, *args: Any, **kwargs: Any) -> Any:
+            pass
+
+        def recycle(self, *args: Any, **kwargs: Any) -> None:
+            pass
+
+    class PooledObject:  # type: ignore
+        def __init__(self, keeped_object: Any = None) -> None:
+            self.keeped_object = keeped_object
+
+    class PooledObjectFactory:  # type: ignore
+        def __init__(self, *args: Any, **kwargs: Any) -> None:
+            pass
 
 T = TypeVar('T')
 
-class GenericPooledObjectFactory(PooledObjectFactory):
+class GenericPooledObjectFactory(PooledObjectFactory):  # type: ignore[misc]
     """Generic factory class for creating pooled objects of any type."""
     
     def __init__(
@@ -79,7 +103,7 @@ def get_object_pool(
     time_between_eviction_runs: int = 300,  # Less frequent eviction to maintain high reuse ratio
     eviction_weight: float = 0.3,  # Less aggressive eviction for better reuse
     prewarm_count: int = 5  # Lower pre-warm count to reduce initial memory usage
-) -> Pond:
+) -> Pond | None:
     """Get or create a global object pool instance with balanced eviction-based memory control.
     
     Memory is controlled through moderate eviction to balance reuse ratio and memory usage:
@@ -98,9 +122,13 @@ def get_object_pool(
         prewarm_count: Number of objects to pre-warm the pool with (default: 5)
     
     Returns:
-        Pond instance for the specified object type
+        Pond instance for the specified object type or None if pond is not available
     """
     
+    # If pond is not available, disable pooling gracefully
+    if not POND_AVAILABLE:
+        return None
+
     if pool_name in _pools:
         return _pools[pool_name]
     
@@ -134,4 +162,4 @@ def _prewarm_pool(pond: Pond, pool_name: str, prewarm_count: int = 20) -> None:
             pond.recycle(pooled_obj, name=f"{pool_name}Factory")
         except Exception:
             # If pre-warming fails, just continue
-            break
\ No newline at end of file
+            break
diff --git a/poetry.lock b/poetry.lock
index a8ef9ee79df..87599768db6 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -2730,9 +2730,10 @@ files = [
 name = "madoka"
 version = "0.7.1"
 description = "Memory-efficient CountMin Sketch key-value structure (based on Madoka C++ library)"
-optional = false
+optional = true
 python-versions = "*"
 groups = ["main"]
+markers = "extra == \"proxy\""
 files = [
     {file = "madoka-0.7.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:7521eee9ace30b376bb54fdcb2cb42bf6b7a0346b0d0b612f25f3299aa4a95af"},
     {file = "madoka-0.7.1.tar.gz", hash = "sha256:e258baa84fc0a3764365993b8bf5e1b065383a6ca8c9f862fb3e3e709843fae7"},
@@ -4044,9 +4045,10 @@ xlsxwriter = ["xlsxwriter"]
 name = "pondpond"
 version = "1.4.1"
 description = "Pond is a high performance object-pooling library for Python."
-optional = false
+optional = true
 python-versions = ">=3.8"
 groups = ["main"]
+markers = "extra == \"proxy\""
 files = [
     {file = "pondpond-1.4.1-py3-none-any.whl", hash = "sha256:641028ead4e8018ca6de1220c660ddd6d6fbf62a60e72f410655dd0451d82880"},
     {file = "pondpond-1.4.1.tar.gz", hash = "sha256:8afa34b869d1434d21dd2ec12644abc3b1733fcda8fcf355300338a13a79bb7b"},
@@ -6747,11 +6749,11 @@ type = ["pytest-mypy"]
 caching = ["diskcache"]
 extra-proxy = ["azure-identity", "azure-keyvault-secrets", "google-cloud-iam", "google-cloud-kms", "prisma", "redisvl", "resend"]
 mlflow = ["mlflow"]
-proxy = ["PyJWT", "apscheduler", "azure-identity", "azure-storage-blob", "backoff", "boto3", "cryptography", "fastapi", "fastapi-sso", "fastuuid", "gunicorn", "litellm-enterprise", "litellm-proxy-extras", "mcp", "orjson", "polars", "pynacl", "python-multipart", "pyyaml", "rich", "rq", "uvicorn", "uvloop", "websockets"]
+proxy = ["PyJWT", "apscheduler", "azure-identity", "azure-storage-blob", "backoff", "boto3", "cryptography", "fastapi", "fastapi-sso", "fastuuid", "gunicorn", "litellm-enterprise", "litellm-proxy-extras", "mcp", "orjson", "polars", "pondpond", "pynacl", "python-multipart", "pyyaml", "rich", "rq", "uvicorn", "uvloop", "websockets"]
 semantic-router = ["semantic-router"]
 utils = ["numpydoc"]
 
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.8.1,<4.0, !=3.9.7"
-content-hash = "75004c6a23b70be86622fa417fd0d62fa3843e6e61c8dff8507ae5c967b7205d"
+content-hash = "16fdc1044b4bb316803cbf1825bc970895526949a8bef93eab741777b7210ca8"
diff --git a/pyproject.toml b/pyproject.toml
index b11b1a1c2e2..4113714415c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -31,7 +31,7 @@ jinja2 = "^3.1.2"
 aiohttp = ">=3.10"
 pydantic = "^2.5.0"
 jsonschema = "^4.22.0"
-pondpond = "^1.4.1"
+pondpond = {version = "^1.4.1", optional = true}
 numpydoc = {version = "*", optional = true} # used in utils.py
 fastuuid = {version = ">=0.12.0", optional = true}
 
@@ -94,6 +94,7 @@ proxy = [
     "rich",
     "polars",
     "fastuuid",
+    "pondpond",
 ]
 
 extra_proxy = [
diff --git a/tests/litellm_utils_tests/test_object_pooling.py b/tests/litellm_utils_tests/test_object_pooling.py
index 5cdc38ee452..8d0272e92b9 100644
--- a/tests/litellm_utils_tests/test_object_pooling.py
+++ b/tests/litellm_utils_tests/test_object_pooling.py
@@ -3,6 +3,7 @@ Simplified tests for object pooling utilities in litellm.
 """
 
 import pytest
+pytest.importorskip("pond")
 
 from litellm.litellm_core_utils.object_pooling import (
     get_object_pool,
@@ -103,4 +104,4 @@ class TestObjectPooling:
 
 
 if __name__ == "__main__":
-    pytest.main([__file__])
\ No newline at end of file
+    pytest.main([__file__])