From 5bf0f7706fb53383657125b99ab68745ce8bc0e8 Mon Sep 17 00:00:00 2001 From: Jeevan Babu Gotru <114808955+jeevan1098@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:46:48 -0500 Subject: [PATCH 1/5] fix(github_copilot): surface copilot_usage token_details as cache token counts --- .../github_copilot/chat/transformation.py | 54 ++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/litellm/llms/github_copilot/chat/transformation.py b/litellm/llms/github_copilot/chat/transformation.py index 83bc8df23a6..2034146b274 100644 --- a/litellm/llms/github_copilot/chat/transformation.py +++ b/litellm/llms/github_copilot/chat/transformation.py @@ -28,6 +28,34 @@ if TYPE_CHECKING: _DECODED_JSON: Final = TypeAdapter(object) +def copilot_cache_token_counts(usage: Mapping) -> tuple[int, int]: + copilot_usage: Final = usage.get("copilot_usage") + if not isinstance(copilot_usage, Mapping): + return 0, 0 + token_details: Final = copilot_usage.get("token_details") + if not isinstance(token_details, list): + return 0, 0 + cache_read: Final = sum( + entry["token_count"] + for entry in token_details + if isinstance(entry, Mapping) + and entry.get("token_type") == "cache_read" + and isinstance(entry.get("token_count"), int) + and not isinstance(entry.get("token_count"), bool) + and entry["token_count"] >= 0 + ) + cache_write: Final = sum( + entry["token_count"] + for entry in token_details + if isinstance(entry, Mapping) + and entry.get("token_type") == "cache_write" + and isinstance(entry.get("token_count"), int) + and not isinstance(entry.get("token_count"), bool) + and entry["token_count"] >= 0 + ) + return cache_read, cache_write + + class GithubCopilotConfig(OpenAIConfig): def __init__( self, @@ -231,6 +259,17 @@ class GithubCopilotConfig(OpenAIConfig): normalized["completion_tokens"] = usage["output_tokens"] if "total_tokens" not in normalized: normalized["total_tokens"] = normalized.get("prompt_tokens", 0) + normalized.get("completion_tokens", 0) + cache_read, cache_write = copilot_cache_token_counts(usage) + if cache_read and "cache_read_input_tokens" not in normalized: + normalized["cache_read_input_tokens"] = cache_read + if cache_write and "cache_creation_input_tokens" not in normalized: + normalized["cache_creation_input_tokens"] = cache_write + if cache_read: + existing_details: Final = normalized.get("prompt_tokens_details") + details: Final = dict(existing_details) if isinstance(existing_details, Mapping) else {} + if not details.get("cached_tokens"): + details["cached_tokens"] = cache_read + normalized["prompt_tokens_details"] = details return normalized @classmethod @@ -296,7 +335,11 @@ class GithubCopilotConfig(OpenAIConfig): Repair the OpenAI-SDK-parsed response on the handler path that bypasses transform_response. See: https://github.com/BerriAI/litellm/issues/30927 """ - return self._synthesize_choices_for_anthropic_native(parsed_response) + repaired: Final = self._synthesize_choices_for_anthropic_native(parsed_response) + usage: Final = repaired.get("usage") + if repaired.get("choices") and isinstance(usage, dict) and "copilot_usage" in usage: + return {**repaired, "usage": self._normalize_anthropic_usage(usage)} + return repaired def transform_response( self, @@ -337,6 +380,15 @@ class GithubCopilotConfig(OpenAIConfig): headers=raw_response.headers, content=json.dumps(synthesized).encode(), ) + else: + usage: Final = response_json.get("usage") + if isinstance(usage, dict) and "copilot_usage" in usage: + rewritten: Final = {**response_json, "usage": self._normalize_anthropic_usage(usage)} + raw_response = httpx.Response( # rebind-ok: replaces the raw body with the normalized usage payload + status_code=raw_response.status_code, + headers=raw_response.headers, + content=json.dumps(rewritten).encode(), + ) return super().transform_response( model=model, From 830314177fd75adb8a513a53b2837ca3fd93f4b3 Mon Sep 17 00:00:00 2001 From: Jeevan Babu Gotru <114808955+jeevan1098@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:20:41 -0500 Subject: [PATCH 2/5] fix(github_copilot): surface copilot_usage token_details as cache token counts --- .../litellm_core_utils/streaming_handler.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 2df470edc59..7c3d56b0245 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -2757,10 +2757,36 @@ def calculate_total_usage(chunks: Sequence[ModelResponse | ModelResponseStream]) ) if latest_cost is not None: returned_usage_chunk.cost = latest_cost + _apply_copilot_cache_counts(returned_usage_chunk, latest_usage_chunk) return returned_usage_chunk +def _apply_copilot_cache_counts(returned_usage: Usage, latest_usage_chunk: Usage | Mapping[str, int]) -> None: + from litellm.llms.github_copilot.chat.transformation import copilot_cache_token_counts + + copilot_usage: Final = ( + latest_usage_chunk.get("copilot_usage") + if isinstance(latest_usage_chunk, Mapping) + else getattr(latest_usage_chunk, "copilot_usage", None) + ) + if not isinstance(copilot_usage, Mapping): + return + cache_read, cache_write = copilot_cache_token_counts({"copilot_usage": copilot_usage}) + if not cache_read and not cache_write: + return + details: Final = returned_usage.prompt_tokens_details or PromptTokensDetailsWrapper() + if cache_read and not getattr(details, "cached_tokens", None): + details.cached_tokens = cache_read + if cache_write and not getattr(details, "cache_write_tokens", None): + details.cache_write_tokens = cache_write + returned_usage.prompt_tokens_details = details # rebind-ok: attaches copilot cache counts to the aggregated usage + if cache_read and getattr(returned_usage, "cache_read_input_tokens", None) is None: + returned_usage.cache_read_input_tokens = cache_read + if cache_write and getattr(returned_usage, "cache_creation_input_tokens", None) is None: + returned_usage.cache_creation_input_tokens = cache_write + + def generic_chunk_has_all_required_fields(chunk: dict) -> bool: """ Checks if the provided chunk dictionary contains all required fields for GenericStreamingChunk. From 9a463793921b27a36033add1f6883f98f04a886e Mon Sep 17 00:00:00 2001 From: Jeevan Babu Gotru <114808955+jeevan1098@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:21:24 -0500 Subject: [PATCH 3/5] fix(github_copilot): surface copilot_usage token_details as cache token counts --- .../test_github_copilot_transformation.py | 173 ++++++++++++++++++ 1 file changed, 173 insertions(+) diff --git a/tests/unit/llms/github_copilot/test_github_copilot_transformation.py b/tests/unit/llms/github_copilot/test_github_copilot_transformation.py index 2dbfae07526..a150a08b811 100644 --- a/tests/unit/llms/github_copilot/test_github_copilot_transformation.py +++ b/tests/unit/llms/github_copilot/test_github_copilot_transformation.py @@ -966,3 +966,176 @@ def test_transform_response_carries_upstream_usage(): assert result.usage.prompt_tokens == 11 assert result.usage.completion_tokens == 7 assert result.usage.total_tokens == 18 + + +class TestGithubCopilotUsageTokenDetails: + """ + The Copilot API reports prompt cache usage only in + usage.copilot_usage.token_details, as cache_read and cache_write entries. + Those counts must surface as cache_read_input_tokens, + cache_creation_input_tokens and prompt_tokens_details.cached_tokens on + every response path. + + See: https://github.com/BerriAI/litellm/issues/48567 + """ + + CACHE_READ = 18002 + CACHE_WRITE = 18002 + + @staticmethod + def _token_details(*entries): + return [{"token_type": token_type, "token_count": count} for token_type, count in entries] + + @staticmethod + def _anthropic_native_response(token_details): + return { + "id": "msg_1", + "model": "claude-sonnet-5", + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 18027, + "output_tokens": 10, + "copilot_usage": {"token_details": token_details}, + }, + } + + @staticmethod + def _openai_response(token_details): + return { + "id": "chatcmpl-1", + "created": 1, + "model": "claude-sonnet-5", + "object": "chat.completion", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 18027, + "completion_tokens": 10, + "total_tokens": 18037, + "copilot_usage": {"token_details": token_details}, + }, + } + + def test_cache_write_surfaces_as_cache_creation_input_tokens_on_anthropic_native_path(self): + config = GithubCopilotConfig() + response = self._anthropic_native_response( + self._token_details(("input", 25), ("cache_write", self.CACHE_WRITE), ("output", 10)) + ) + + usage = config.transform_parsed_response_dict(response)["usage"] + + assert usage["cache_creation_input_tokens"] == self.CACHE_WRITE + + def test_cache_read_surfaces_as_cache_read_input_tokens_and_cached_tokens_on_anthropic_native_path(self): + config = GithubCopilotConfig() + response = self._anthropic_native_response(self._token_details(("cache_read", self.CACHE_READ))) + + usage = config.transform_parsed_response_dict(response)["usage"] + + assert usage["cache_read_input_tokens"] == self.CACHE_READ + assert usage["prompt_tokens_details"]["cached_tokens"] == self.CACHE_READ + + def test_cache_counts_surface_on_openai_style_parsed_response(self): + config = GithubCopilotConfig() + response = self._openai_response( + self._token_details(("cache_read", self.CACHE_READ), ("cache_write", self.CACHE_WRITE)) + ) + + usage = config.transform_parsed_response_dict(response)["usage"] + + assert usage["cache_read_input_tokens"] == self.CACHE_READ + assert usage["cache_creation_input_tokens"] == self.CACHE_WRITE + assert usage["prompt_tokens_details"]["cached_tokens"] == self.CACHE_READ + + def test_cache_counts_reach_model_response_usage_through_transform_response(self): + config = GithubCopilotConfig() + body = self._openai_response( + self._token_details(("cache_read", self.CACHE_READ), ("cache_write", self.CACHE_WRITE)) + ) + raw_response = httpx.Response( + status_code=200, + headers={"content-type": "application/json"}, + content=json.dumps(body).encode(), + ) + + model_response = config.transform_response( + model="github_copilot/claude-sonnet-5", + raw_response=raw_response, + model_response=ModelResponse(), + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + api_key="test", + ) + + usage = model_response.usage + assert usage is not None + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.cached_tokens == self.CACHE_READ + assert usage.prompt_tokens_details.cache_write_tokens == self.CACHE_WRITE + assert usage["cache_read_input_tokens"] == self.CACHE_READ + assert usage["cache_creation_input_tokens"] == self.CACHE_WRITE + + def test_cache_counts_reach_aggregated_streaming_usage(self): + from litellm.litellm_core_utils.streaming_handler import calculate_total_usage + from litellm.types.utils import ModelResponseStream + + chunk = ModelResponseStream( + **{ + "id": "chatcmpl-1", + "created": 1, + "model": "claude-sonnet-5", + "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": {"content": ""}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 18027, + "completion_tokens": 10, + "total_tokens": 18037, + "copilot_usage": { + "token_details": self._token_details( + ("cache_read", self.CACHE_READ), ("cache_write", self.CACHE_WRITE) + ) + }, + }, + } + ) + + usage = calculate_total_usage(chunks=[chunk]) + + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.cached_tokens == self.CACHE_READ + assert usage.prompt_tokens_details.cache_write_tokens == self.CACHE_WRITE + assert usage["cache_read_input_tokens"] == self.CACHE_READ + assert usage["cache_creation_input_tokens"] == self.CACHE_WRITE + + def test_repeated_token_detail_entries_are_summed_and_malformed_entries_ignored(self): + config = GithubCopilotConfig() + token_details = self._token_details(("cache_read", 100), ("cache_read", 23)) + [ + {"token_type": "cache_read"}, + {"token_type": "cache_write", "token_count": -5}, + {"token_type": "cache_write", "token_count": "18002"}, + {"token_count": 10}, + "not-a-dict", + ] + response = self._anthropic_native_response(token_details) + + usage = config.transform_parsed_response_dict(response)["usage"] + + assert usage["cache_read_input_tokens"] == 123 + assert "cache_creation_input_tokens" not in usage + + def test_provider_reported_cache_fields_are_not_overwritten(self): + config = GithubCopilotConfig() + response = self._anthropic_native_response( + self._token_details(("cache_read", self.CACHE_READ), ("cache_write", self.CACHE_WRITE)) + ) + response["usage"]["cache_read_input_tokens"] = 7 + response["usage"]["cache_creation_input_tokens"] = 9 + + usage = config.transform_parsed_response_dict(response)["usage"] + + assert usage["cache_read_input_tokens"] == 7 + assert usage["cache_creation_input_tokens"] == 9 From 4909bfd15190e3c21d08ab9c1eee28298bffc5a8 Mon Sep 17 00:00:00 2001 From: Jeevan Babu Gotru <114808955+jeevan1098@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:26:48 -0500 Subject: [PATCH 4/5] fix(github_copilot): suppress LIT011 on copilot cache count assignments in streaming handler --- litellm/litellm_core_utils/streaming_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 7c3d56b0245..91abb5044d9 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -2782,9 +2782,9 @@ def _apply_copilot_cache_counts(returned_usage: Usage, latest_usage_chunk: Usage details.cache_write_tokens = cache_write returned_usage.prompt_tokens_details = details # rebind-ok: attaches copilot cache counts to the aggregated usage if cache_read and getattr(returned_usage, "cache_read_input_tokens", None) is None: - returned_usage.cache_read_input_tokens = cache_read + returned_usage.cache_read_input_tokens = cache_read # rebind-ok: attaches copilot cache counts to the aggregated usage if cache_write and getattr(returned_usage, "cache_creation_input_tokens", None) is None: - returned_usage.cache_creation_input_tokens = cache_write + returned_usage.cache_creation_input_tokens = cache_write # rebind-ok: attaches copilot cache counts to the aggregated usage def generic_chunk_has_all_required_fields(chunk: dict) -> bool: From 641f823813068b9c6c13ee2435ad0263792b8b92 Mon Sep 17 00:00:00 2001 From: Jeevan Babu Gotru <114808955+jeevan1098@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:04:31 -0500 Subject: [PATCH 5/5] fix(github_copilot): wrap copilot cache count assignments for ruff format --- litellm/litellm_core_utils/streaming_handler.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 91abb5044d9..b8e559d7103 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -2782,9 +2782,13 @@ def _apply_copilot_cache_counts(returned_usage: Usage, latest_usage_chunk: Usage details.cache_write_tokens = cache_write returned_usage.prompt_tokens_details = details # rebind-ok: attaches copilot cache counts to the aggregated usage if cache_read and getattr(returned_usage, "cache_read_input_tokens", None) is None: - returned_usage.cache_read_input_tokens = cache_read # rebind-ok: attaches copilot cache counts to the aggregated usage + returned_usage.cache_read_input_tokens = ( # rebind-ok: attaches copilot cache counts to the aggregated usage + cache_read + ) if cache_write and getattr(returned_usage, "cache_creation_input_tokens", None) is None: - returned_usage.cache_creation_input_tokens = cache_write # rebind-ok: attaches copilot cache counts to the aggregated usage + returned_usage.cache_creation_input_tokens = ( # rebind-ok: attaches copilot cache counts to the aggregated usage + cache_write + ) def generic_chunk_has_all_required_fields(chunk: dict) -> bool: