scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-10 08:28:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 108

Merge 641f823813 into cfc651dcd5

Browse source
This commit is contained in:
Jeevan Babu Gotru 2026-10-28 15:04:36 +00:00 • committed by GitHub
commit a3cff7c7d7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 256 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
litellm/litellm_core_utils/streaming_handler.py
View file

@ -2762,10 +2762,40 @@ def calculate_total_usage(chunks: Sequence[ModelResponse | ModelResponseStream])
)
if latest_cost is not None:
returned_usage_chunk.cost = latest_cost
_apply_copilot_cache_counts(returned_usage_chunk, latest_usage_chunk)
return returned_usage_chunk
def _apply_copilot_cache_counts(returned_usage: Usage, latest_usage_chunk: Usage | Mapping[str, int]) -> None:
from litellm.llms.github_copilot.chat.transformation import copilot_cache_token_counts
copilot_usage: Final = (
latest_usage_chunk.get("copilot_usage")
if isinstance(latest_usage_chunk, Mapping)
else getattr(latest_usage_chunk, "copilot_usage", None)
)
if not isinstance(copilot_usage, Mapping):
return
cache_read, cache_write = copilot_cache_token_counts({"copilot_usage": copilot_usage})
if not cache_read and not cache_write:
return
details: Final = returned_usage.prompt_tokens_details or PromptTokensDetailsWrapper()
if cache_read and not getattr(details, "cached_tokens", None):
details.cached_tokens = cache_read
if cache_write and not getattr(details, "cache_write_tokens", None):
details.cache_write_tokens = cache_write
returned_usage.prompt_tokens_details = details # rebind-ok: attaches copilot cache counts to the aggregated usage
if cache_read and getattr(returned_usage, "cache_read_input_tokens", None) is None:
returned_usage.cache_read_input_tokens = ( # rebind-ok: attaches copilot cache counts to the aggregated usage
cache_read
)
if cache_write and getattr(returned_usage, "cache_creation_input_tokens", None) is None:
returned_usage.cache_creation_input_tokens = ( # rebind-ok: attaches copilot cache counts to the aggregated usage
cache_write
)
def generic_chunk_has_all_required_fields(chunk: dict) -> bool:
"""
Checks if the provided chunk dictionary contains all required fields for GenericStreamingChunk.

54
litellm/llms/github_copilot/chat/transformation.py
View file

@ -28,6 +28,34 @@ if TYPE_CHECKING:
_DECODED_JSON: Final = TypeAdapter(object)
def copilot_cache_token_counts(usage: Mapping) -> tuple[int, int]:
copilot_usage: Final = usage.get("copilot_usage")
if not isinstance(copilot_usage, Mapping):
return 0, 0
token_details: Final = copilot_usage.get("token_details")
if not isinstance(token_details, list):
return 0, 0
cache_read: Final = sum(
entry["token_count"]
for entry in token_details
if isinstance(entry, Mapping)
and entry.get("token_type") == "cache_read"
and isinstance(entry.get("token_count"), int)
and not isinstance(entry.get("token_count"), bool)
and entry["token_count"] >= 0
)
cache_write: Final = sum(
entry["token_count"]
for entry in token_details
if isinstance(entry, Mapping)
and entry.get("token_type") == "cache_write"
and isinstance(entry.get("token_count"), int)
and not isinstance(entry.get("token_count"), bool)
and entry["token_count"] >= 0
)
return cache_read, cache_write
class GithubCopilotConfig(OpenAIConfig):
def __init__(
self,
@ -231,6 +259,17 @@ class GithubCopilotConfig(OpenAIConfig):
normalized["completion_tokens"] = usage["output_tokens"]
if "total_tokens" not in normalized:
normalized["total_tokens"] = normalized.get("prompt_tokens", 0) + normalized.get("completion_tokens", 0)
cache_read, cache_write = copilot_cache_token_counts(usage)
if cache_read and "cache_read_input_tokens" not in normalized:
normalized["cache_read_input_tokens"] = cache_read
if cache_write and "cache_creation_input_tokens" not in normalized:
normalized["cache_creation_input_tokens"] = cache_write
if cache_read:
existing_details: Final = normalized.get("prompt_tokens_details")
details: Final = dict(existing_details) if isinstance(existing_details, Mapping) else {}
if not details.get("cached_tokens"):
details["cached_tokens"] = cache_read
normalized["prompt_tokens_details"] = details
return normalized
@classmethod
@ -296,7 +335,11 @@ class GithubCopilotConfig(OpenAIConfig):
Repair the OpenAI-SDK-parsed response on the handler path that bypasses
transform_response. See: https://github.com/BerriAI/litellm/issues/30927
"""
return self._synthesize_choices_for_anthropic_native(parsed_response)
repaired: Final = self._synthesize_choices_for_anthropic_native(parsed_response)
usage: Final = repaired.get("usage")
if repaired.get("choices") and isinstance(usage, dict) and "copilot_usage" in usage:
return {**repaired, "usage": self._normalize_anthropic_usage(usage)}
return repaired
def transform_response(
self,
@ -337,6 +380,15 @@ class GithubCopilotConfig(OpenAIConfig):
headers=raw_response.headers,
content=json.dumps(synthesized).encode(),
)
else:
usage: Final = response_json.get("usage")
if isinstance(usage, dict) and "copilot_usage" in usage:
rewritten: Final = {**response_json, "usage": self._normalize_anthropic_usage(usage)}
raw_response = httpx.Response( # rebind-ok: replaces the raw body with the normalized usage payload
status_code=raw_response.status_code,
headers=raw_response.headers,
content=json.dumps(rewritten).encode(),
)
return super().transform_response(
model=model,

173
tests/unit/llms/github_copilot/test_github_copilot_transformation.py
View file

@ -966,3 +966,176 @@ def test_transform_response_carries_upstream_usage():
assert result.usage.prompt_tokens == 11
assert result.usage.completion_tokens == 7
assert result.usage.total_tokens == 18
class TestGithubCopilotUsageTokenDetails:
"""
The Copilot API reports prompt cache usage only in
usage.copilot_usage.token_details, as cache_read and cache_write entries.
Those counts must surface as cache_read_input_tokens,
cache_creation_input_tokens and prompt_tokens_details.cached_tokens on
every response path.
See: https://github.com/BerriAI/litellm/issues/48567
"""
CACHE_READ = 18002
CACHE_WRITE = 18002
@staticmethod
def _token_details(*entries):
return [{"token_type": token_type, "token_count": count} for token_type, count in entries]
@staticmethod
def _anthropic_native_response(token_details):
return {
"id": "msg_1",
"model": "claude-sonnet-5",
"content": [{"type": "text", "text": "hi"}],
"stop_reason": "end_turn",
"usage": {
"input_tokens": 18027,
"output_tokens": 10,
"copilot_usage": {"token_details": token_details},
},
}
@staticmethod
def _openai_response(token_details):
return {
"id": "chatcmpl-1",
"created": 1,
"model": "claude-sonnet-5",
"object": "chat.completion",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
"usage": {
"prompt_tokens": 18027,
"completion_tokens": 10,
"total_tokens": 18037,
"copilot_usage": {"token_details": token_details},
},
}
def test_cache_write_surfaces_as_cache_creation_input_tokens_on_anthropic_native_path(self):
config = GithubCopilotConfig()
response = self._anthropic_native_response(
self._token_details(("input", 25), ("cache_write", self.CACHE_WRITE), ("output", 10))
)
usage = config.transform_parsed_response_dict(response)["usage"]
assert usage["cache_creation_input_tokens"] == self.CACHE_WRITE
def test_cache_read_surfaces_as_cache_read_input_tokens_and_cached_tokens_on_anthropic_native_path(self):
config = GithubCopilotConfig()
response = self._anthropic_native_response(self._token_details(("cache_read", self.CACHE_READ)))
usage = config.transform_parsed_response_dict(response)["usage"]
assert usage["cache_read_input_tokens"] == self.CACHE_READ
assert usage["prompt_tokens_details"]["cached_tokens"] == self.CACHE_READ
def test_cache_counts_surface_on_openai_style_parsed_response(self):
config = GithubCopilotConfig()
response = self._openai_response(
self._token_details(("cache_read", self.CACHE_READ), ("cache_write", self.CACHE_WRITE))
)
usage = config.transform_parsed_response_dict(response)["usage"]
assert usage["cache_read_input_tokens"] == self.CACHE_READ
assert usage["cache_creation_input_tokens"] == self.CACHE_WRITE
assert usage["prompt_tokens_details"]["cached_tokens"] == self.CACHE_READ
def test_cache_counts_reach_model_response_usage_through_transform_response(self):
config = GithubCopilotConfig()
body = self._openai_response(
self._token_details(("cache_read", self.CACHE_READ), ("cache_write", self.CACHE_WRITE))
)
raw_response = httpx.Response(
status_code=200,
headers={"content-type": "application/json"},
content=json.dumps(body).encode(),
)
model_response = config.transform_response(
model="github_copilot/claude-sonnet-5",
raw_response=raw_response,
model_response=ModelResponse(),
logging_obj=MagicMock(),
request_data={},
messages=[],
optional_params={},
litellm_params={},
encoding=None,
api_key="test",
)
usage = model_response.usage
assert usage is not None
assert usage.prompt_tokens_details is not None
assert usage.prompt_tokens_details.cached_tokens == self.CACHE_READ
assert usage.prompt_tokens_details.cache_write_tokens == self.CACHE_WRITE
assert usage["cache_read_input_tokens"] == self.CACHE_READ
assert usage["cache_creation_input_tokens"] == self.CACHE_WRITE
def test_cache_counts_reach_aggregated_streaming_usage(self):
from litellm.litellm_core_utils.streaming_handler import calculate_total_usage
from litellm.types.utils import ModelResponseStream
chunk = ModelResponseStream(
**{
"id": "chatcmpl-1",
"created": 1,
"model": "claude-sonnet-5",
"object": "chat.completion.chunk",
"choices": [{"index": 0, "delta": {"content": ""}, "finish_reason": "stop"}],
"usage": {
"prompt_tokens": 18027,
"completion_tokens": 10,
"total_tokens": 18037,
"copilot_usage": {
"token_details": self._token_details(
("cache_read", self.CACHE_READ), ("cache_write", self.CACHE_WRITE)
)
},
},
}
)
usage = calculate_total_usage(chunks=[chunk])
assert usage.prompt_tokens_details is not None
assert usage.prompt_tokens_details.cached_tokens == self.CACHE_READ
assert usage.prompt_tokens_details.cache_write_tokens == self.CACHE_WRITE
assert usage["cache_read_input_tokens"] == self.CACHE_READ
assert usage["cache_creation_input_tokens"] == self.CACHE_WRITE
def test_repeated_token_detail_entries_are_summed_and_malformed_entries_ignored(self):
config = GithubCopilotConfig()
token_details = self._token_details(("cache_read", 100), ("cache_read", 23)) + [
{"token_type": "cache_read"},
{"token_type": "cache_write", "token_count": -5},
{"token_type": "cache_write", "token_count": "18002"},
{"token_count": 10},
"not-a-dict",
]
response = self._anthropic_native_response(token_details)
usage = config.transform_parsed_response_dict(response)["usage"]
assert usage["cache_read_input_tokens"] == 123
assert "cache_creation_input_tokens" not in usage
def test_provider_reported_cache_fields_are_not_overwritten(self):
config = GithubCopilotConfig()
response = self._anthropic_native_response(
self._token_details(("cache_read", self.CACHE_READ), ("cache_write", self.CACHE_WRITE))
)
response["usage"]["cache_read_input_tokens"] = 7
response["usage"]["cache_creation_input_tokens"] = 9
usage = config.transform_parsed_response_dict(response)["usage"]
assert usage["cache_read_input_tokens"] == 7
assert usage["cache_creation_input_tokens"] == 9