diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 098a404ea14..5100df7e98c 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -2822,10 +2822,34 @@ def calculate_total_usage(chunks: Sequence[ModelResponse | ModelResponseStream]) ) if latest_cost is not None: returned_usage_chunk.cost = latest_cost + _apply_copilot_cache_counts(returned_usage_chunk, latest_usage_chunk) return returned_usage_chunk +def _apply_copilot_cache_counts(returned_usage: Usage, latest_usage_chunk: Usage | Mapping[str, int]) -> None: + from litellm.llms.github_copilot.chat.transformation import copilot_cache_token_counts + + copilot_usage: Final = ( + latest_usage_chunk.get("copilot_usage") + if isinstance(latest_usage_chunk, Mapping) + else getattr(latest_usage_chunk, "copilot_usage", None) + ) + if not isinstance(copilot_usage, Mapping): + return + copilot_usage_map: Mapping[str, object] = copilot_usage + cache_read, cache_write = copilot_cache_token_counts({"copilot_usage": copilot_usage_map}) + if not cache_read and not cache_write: + return + details: Final = returned_usage.prompt_tokens_details or PromptTokensDetailsWrapper() + if cache_read and not getattr(details, "cached_tokens", None): + details.cached_tokens = cache_read + if cache_write and not getattr(details, "cache_write_tokens", None): + details.cache_write_tokens = cache_write + returned_usage.prompt_tokens_details = details # rebind-ok: attaches copilot cache counts to the aggregated usage + backfill_missing_cache_usage_fields(returned_usage) + + def generic_chunk_has_all_required_fields(chunk: dict) -> bool: """ Checks if the provided chunk dictionary contains all required fields for GenericStreamingChunk. diff --git a/litellm/llms/github_copilot/chat/transformation.py b/litellm/llms/github_copilot/chat/transformation.py index 83bc8df23a6..780612ed84f 100644 --- a/litellm/llms/github_copilot/chat/transformation.py +++ b/litellm/llms/github_copilot/chat/transformation.py @@ -28,6 +28,32 @@ if TYPE_CHECKING: _DECODED_JSON: Final = TypeAdapter(object) +def _copilot_token_detail_count(entry: object, token_type: str) -> int: + if not isinstance(entry, Mapping): + return 0 + entry_map: Mapping[str, object] = entry + if entry_map.get("token_type") != token_type: + return 0 + token_count: Final = entry_map.get("token_count") + if isinstance(token_count, int) and not isinstance(token_count, bool) and token_count >= 0: + return token_count + return 0 + + +def copilot_cache_token_counts(usage: Mapping[str, object]) -> tuple[int, int]: + copilot_usage: Final = usage.get("copilot_usage") + if not isinstance(copilot_usage, Mapping): + return 0, 0 + copilot_usage_map: Mapping[str, object] = copilot_usage + token_details: Final = copilot_usage_map.get("token_details") + if not isinstance(token_details, list): + return 0, 0 + token_detail_entries: list[object] = token_details + cache_read: Final = sum(_copilot_token_detail_count(entry, "cache_read") for entry in token_detail_entries) + cache_write: Final = sum(_copilot_token_detail_count(entry, "cache_write") for entry in token_detail_entries) + return cache_read, cache_write + + class GithubCopilotConfig(OpenAIConfig): def __init__( self, @@ -231,6 +257,17 @@ class GithubCopilotConfig(OpenAIConfig): normalized["completion_tokens"] = usage["output_tokens"] if "total_tokens" not in normalized: normalized["total_tokens"] = normalized.get("prompt_tokens", 0) + normalized.get("completion_tokens", 0) + cache_read, cache_write = copilot_cache_token_counts(usage) + if cache_read and "cache_read_input_tokens" not in normalized: + normalized["cache_read_input_tokens"] = cache_read + if cache_write and "cache_creation_input_tokens" not in normalized: + normalized["cache_creation_input_tokens"] = cache_write + if cache_read: + existing_details: Final = normalized.get("prompt_tokens_details") + details: Final = dict(existing_details) if isinstance(existing_details, Mapping) else {} + if not details.get("cached_tokens"): + details["cached_tokens"] = cache_read + normalized["prompt_tokens_details"] = details return normalized @classmethod @@ -296,7 +333,11 @@ class GithubCopilotConfig(OpenAIConfig): Repair the OpenAI-SDK-parsed response on the handler path that bypasses transform_response. See: https://github.com/BerriAI/litellm/issues/30927 """ - return self._synthesize_choices_for_anthropic_native(parsed_response) + repaired: Final = self._synthesize_choices_for_anthropic_native(parsed_response) + usage: Final = repaired.get("usage") + if repaired.get("choices") and isinstance(usage, dict) and "copilot_usage" in usage: + return {**repaired, "usage": self._normalize_anthropic_usage(usage)} + return repaired def transform_response( self, @@ -337,6 +378,15 @@ class GithubCopilotConfig(OpenAIConfig): headers=raw_response.headers, content=json.dumps(synthesized).encode(), ) + else: + usage: Final = response_json.get("usage") + if isinstance(usage, dict) and "copilot_usage" in usage: + rewritten: Final = {**response_json, "usage": self._normalize_anthropic_usage(usage)} + raw_response = httpx.Response( # rebind-ok: replaces the raw body with the normalized usage payload + status_code=raw_response.status_code, + headers=raw_response.headers, + content=json.dumps(rewritten).encode(), + ) return super().transform_response( model=model, diff --git a/tests/unit/llms/github_copilot/test_github_copilot_transformation.py b/tests/unit/llms/github_copilot/test_github_copilot_transformation.py index 2dbfae07526..a150a08b811 100644 --- a/tests/unit/llms/github_copilot/test_github_copilot_transformation.py +++ b/tests/unit/llms/github_copilot/test_github_copilot_transformation.py @@ -966,3 +966,176 @@ def test_transform_response_carries_upstream_usage(): assert result.usage.prompt_tokens == 11 assert result.usage.completion_tokens == 7 assert result.usage.total_tokens == 18 + + +class TestGithubCopilotUsageTokenDetails: + """ + The Copilot API reports prompt cache usage only in + usage.copilot_usage.token_details, as cache_read and cache_write entries. + Those counts must surface as cache_read_input_tokens, + cache_creation_input_tokens and prompt_tokens_details.cached_tokens on + every response path. + + See: https://github.com/BerriAI/litellm/issues/48567 + """ + + CACHE_READ = 18002 + CACHE_WRITE = 18002 + + @staticmethod + def _token_details(*entries): + return [{"token_type": token_type, "token_count": count} for token_type, count in entries] + + @staticmethod + def _anthropic_native_response(token_details): + return { + "id": "msg_1", + "model": "claude-sonnet-5", + "content": [{"type": "text", "text": "hi"}], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 18027, + "output_tokens": 10, + "copilot_usage": {"token_details": token_details}, + }, + } + + @staticmethod + def _openai_response(token_details): + return { + "id": "chatcmpl-1", + "created": 1, + "model": "claude-sonnet-5", + "object": "chat.completion", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 18027, + "completion_tokens": 10, + "total_tokens": 18037, + "copilot_usage": {"token_details": token_details}, + }, + } + + def test_cache_write_surfaces_as_cache_creation_input_tokens_on_anthropic_native_path(self): + config = GithubCopilotConfig() + response = self._anthropic_native_response( + self._token_details(("input", 25), ("cache_write", self.CACHE_WRITE), ("output", 10)) + ) + + usage = config.transform_parsed_response_dict(response)["usage"] + + assert usage["cache_creation_input_tokens"] == self.CACHE_WRITE + + def test_cache_read_surfaces_as_cache_read_input_tokens_and_cached_tokens_on_anthropic_native_path(self): + config = GithubCopilotConfig() + response = self._anthropic_native_response(self._token_details(("cache_read", self.CACHE_READ))) + + usage = config.transform_parsed_response_dict(response)["usage"] + + assert usage["cache_read_input_tokens"] == self.CACHE_READ + assert usage["prompt_tokens_details"]["cached_tokens"] == self.CACHE_READ + + def test_cache_counts_surface_on_openai_style_parsed_response(self): + config = GithubCopilotConfig() + response = self._openai_response( + self._token_details(("cache_read", self.CACHE_READ), ("cache_write", self.CACHE_WRITE)) + ) + + usage = config.transform_parsed_response_dict(response)["usage"] + + assert usage["cache_read_input_tokens"] == self.CACHE_READ + assert usage["cache_creation_input_tokens"] == self.CACHE_WRITE + assert usage["prompt_tokens_details"]["cached_tokens"] == self.CACHE_READ + + def test_cache_counts_reach_model_response_usage_through_transform_response(self): + config = GithubCopilotConfig() + body = self._openai_response( + self._token_details(("cache_read", self.CACHE_READ), ("cache_write", self.CACHE_WRITE)) + ) + raw_response = httpx.Response( + status_code=200, + headers={"content-type": "application/json"}, + content=json.dumps(body).encode(), + ) + + model_response = config.transform_response( + model="github_copilot/claude-sonnet-5", + raw_response=raw_response, + model_response=ModelResponse(), + logging_obj=MagicMock(), + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + api_key="test", + ) + + usage = model_response.usage + assert usage is not None + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.cached_tokens == self.CACHE_READ + assert usage.prompt_tokens_details.cache_write_tokens == self.CACHE_WRITE + assert usage["cache_read_input_tokens"] == self.CACHE_READ + assert usage["cache_creation_input_tokens"] == self.CACHE_WRITE + + def test_cache_counts_reach_aggregated_streaming_usage(self): + from litellm.litellm_core_utils.streaming_handler import calculate_total_usage + from litellm.types.utils import ModelResponseStream + + chunk = ModelResponseStream( + **{ + "id": "chatcmpl-1", + "created": 1, + "model": "claude-sonnet-5", + "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": {"content": ""}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 18027, + "completion_tokens": 10, + "total_tokens": 18037, + "copilot_usage": { + "token_details": self._token_details( + ("cache_read", self.CACHE_READ), ("cache_write", self.CACHE_WRITE) + ) + }, + }, + } + ) + + usage = calculate_total_usage(chunks=[chunk]) + + assert usage.prompt_tokens_details is not None + assert usage.prompt_tokens_details.cached_tokens == self.CACHE_READ + assert usage.prompt_tokens_details.cache_write_tokens == self.CACHE_WRITE + assert usage["cache_read_input_tokens"] == self.CACHE_READ + assert usage["cache_creation_input_tokens"] == self.CACHE_WRITE + + def test_repeated_token_detail_entries_are_summed_and_malformed_entries_ignored(self): + config = GithubCopilotConfig() + token_details = self._token_details(("cache_read", 100), ("cache_read", 23)) + [ + {"token_type": "cache_read"}, + {"token_type": "cache_write", "token_count": -5}, + {"token_type": "cache_write", "token_count": "18002"}, + {"token_count": 10}, + "not-a-dict", + ] + response = self._anthropic_native_response(token_details) + + usage = config.transform_parsed_response_dict(response)["usage"] + + assert usage["cache_read_input_tokens"] == 123 + assert "cache_creation_input_tokens" not in usage + + def test_provider_reported_cache_fields_are_not_overwritten(self): + config = GithubCopilotConfig() + response = self._anthropic_native_response( + self._token_details(("cache_read", self.CACHE_READ), ("cache_write", self.CACHE_WRITE)) + ) + response["usage"]["cache_read_input_tokens"] = 7 + response["usage"]["cache_creation_input_tokens"] = 9 + + usage = config.transform_parsed_response_dict(response)["usage"] + + assert usage["cache_read_input_tokens"] == 7 + assert usage["cache_creation_input_tokens"] == 9