This commit is contained in:
mrinal-berri 2026-10-26 11:42:52 -07:00 • committed by GitHub
commit 34a476ce01
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 440 additions and 7 deletions

View file

@ -148,6 +148,7 @@ from .streaming_handler import PassThroughStreamingHandler
from .success_handler import PassThroughEndpointLogging
from .upstream_usage_headers import (
UpstreamReportedUsage,
apply_upstream_reported_model,
apply_upstream_reported_usage,
)
@ -852,6 +853,7 @@ def _build_passthrough_failure_request_payload(
logging_obj: LiteLLMLoggingObj | None,
custom_llm_provider: str | None,
upstream_usage: UpstreamReportedUsage | None = None,
upstream_model: str | None = None,
) -> dict:
"""Build the ``request_data`` dict passed to ``post_call_failure_hook``.
@ -862,6 +864,8 @@ def _build_passthrough_failure_request_payload(
``upstream_usage`` carries the cost and tokens an upstream reported on an
error response. Spend tracking only attributes a recovered cost when it
comes paired with a usage object, so both keys are written together.
``upstream_model`` is the model name the upstream reported, which replaces
the body's ``model`` on the failure row just as it does on a success row.
"""
request_payload: Final[dict] = dict(cast(Mapping[str, object], parsed_body or {})) # cast-ok: json body
if kwargs:
@ -875,6 +879,8 @@ def _build_passthrough_failure_request_payload(
if upstream_usage is not None:
request_payload["response_cost"] = upstream_usage.response_cost or 0.0
request_payload["combined_usage_object"] = Usage(total_tokens=upstream_usage.total_tokens or 0)
if upstream_model is not None:
request_payload["model"] = upstream_model
return request_payload
@ -1538,6 +1544,10 @@ async def pass_through_request(
logging_obj=logging_obj,
headers=response.headers,
)
upstream_model = apply_upstream_reported_model( # rebind-ok: bound once per branch, like upstream_usage
logging_obj=logging_obj,
headers=response.headers,
)
relay_response: Final = await _log_passthrough_upstream_failure(
response=response,
@ -1548,6 +1558,7 @@ async def pass_through_request(
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
upstream_usage=upstream_usage,
upstream_model=upstream_model,
),
logging_obj=logging_obj,
)
@ -1626,6 +1637,10 @@ async def pass_through_request(
logging_obj=logging_obj,
headers=response.headers,
)
upstream_model = apply_upstream_reported_model( # rebind-ok: bound once per branch, like upstream_usage
logging_obj=logging_obj,
headers=response.headers,
)
if _is_streaming_response(response) is True:
logging_obj.stream = True
@ -1640,6 +1655,7 @@ async def pass_through_request(
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
upstream_usage=upstream_usage,
upstream_model=upstream_model,
),
logging_obj=logging_obj,
)
@ -1740,6 +1756,7 @@ async def pass_through_request(
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
upstream_usage=upstream_usage,
upstream_model=upstream_model,
)
failure_request_payload["response_body"] = response_body
await _log_passthrough_upstream_failure(

View file

@ -3,7 +3,9 @@
Some pass-through targets invoke several models internally, so LiteLLM cannot
price the request from the response body. Those targets report the totals for
the whole HTTP request in ``x-litellm-*`` response headers instead; LiteLLM
records the reported values without recomputing them.
records the reported values without recomputing them. A target may also name
the request in ``x-litellm-upstream-model`` so its spend can be told apart from
other requests that carry no ``model`` in the body.
"""
import math
@ -14,11 +16,15 @@ from typing import Final
import httpx
from litellm._logging import verbose_proxy_logger
from litellm.constants import MAX_SPEND_LOG_MODEL_NAME_LENGTH
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.utils import Usage
UPSTREAM_RESPONSE_COST_HEADER: Final = "x-litellm-response-cost"
UPSTREAM_TOTAL_TOKENS_HEADER: Final = "x-litellm-total-tokens"
# Deliberately not x-litellm-model-name/-id/-group: the gateway already sends
# those to clients, and pass-through relays upstream headers back to the client.
UPSTREAM_MODEL_HEADER: Final = "x-litellm-upstream-model"
# model_call_details key holding what the upstream reported, so later stages of
# the success path can tell an upstream-reported cost apart from one LiteLLM
@ -128,6 +134,39 @@ def apply_upstream_reported_usage(
return reported
def parse_upstream_reported_model(headers: httpx.Headers) -> str | None:
"""Read the model name an upstream reported, or ``None`` when it sent none.
An empty or whitespace-only value counts as absent. A repeated header reads
as one comma-separated list (RFC 9110 field combining, which is how httpx
joins it), so only its first element is used. The name is cut to the spend
log's model-name limit, since it becomes a spend row, rollup key and metric
label.
"""
raw_model: Final[str | None] = headers.get(UPSTREAM_MODEL_HEADER)
model: Final = raw_model.split(",", 1)[0].strip() if raw_model is not None else ""
return model[:MAX_SPEND_LOG_MODEL_NAME_LENGTH].rstrip() or None
def apply_upstream_reported_model(
logging_obj: LiteLLMLoggingObj,
headers: httpx.Headers,
) -> str | None:
"""Record the upstream's model name as the request's model.
The name wins over a ``model`` in the request body. It is a label only:
it never marks the request as upstream-priced (see
``has_upstream_reported_usage``) and nothing prices the request from it,
so a name outside the price map leaves the recorded cost alone.
"""
reported_model: Final = parse_upstream_reported_model(headers)
if reported_model is None:
return None
logging_obj.model = reported_model # rebind-ok: the request's logging object is where the recorded model lives
logging_obj.model_call_details["model"] = reported_model # rebind-ok: read by the standard logging payload
return reported_model
def has_upstream_reported_usage(logging_obj: LiteLLMLoggingObj) -> bool:
"""Whether the upstream spoke this contract on the request's response.

View file

@ -0,0 +1,228 @@
import json
import uuid
from collections.abc import Callable, Mapping
from datetime import datetime, timedelta, timezone
from hashlib import sha256
from typing import Final
import pytest
from integration._support.client import Gateway, JsonValue, Scenario, eventually, object_value
from integration._support.database import read_rows
from integration._support.wire import Reply, Request, wire_server
from pydantic import TypeAdapter
pytestmark: Final = pytest.mark.shares_worker_state
_REPORTED_MODEL: Final = "support-summarizer"
_REPORTED_COST: Final = 0.0123
_REPORTED_TOKENS: Final = 4210
_USAGE_HEADERS: Final = {
"x-litellm-response-cost": str(_REPORTED_COST),
"x-litellm-total-tokens": str(_REPORTED_TOKENS),
}
_MODEL_HEADER: Final = {"x-litellm-upstream-model": _REPORTED_MODEL}
def _upstream(
marker: str, headers: Mapping[str, str], *, status: int = 200, stream: bool = False
) -> Callable[[Request], Reply]:
def respond(request: Request) -> Reply:
assert request.method == "POST", request
assert json.loads(request.body)["ticket_id"] == marker, request
if stream:
return Reply(
status=status,
content_type="text/event-stream",
chunks=(f'data: {{"delta": "{marker}"}}\n\n'.encode(), b"data: [DONE]\n\n"),
headers=headers,
)
return Reply(status=status, body=json.dumps({"summary": marker}).encode(), headers=headers)
return respond
def _endpoint(gateway: Gateway, scenario: Scenario, marker: str, target: str, **fields: JsonValue) -> str:
path: Final = f"/support-ai-{marker}"
created: Final = gateway.post(
"/config/pass_through_endpoint", {"path": path, "target": target, "auth": True, **fields}
)
endpoint_id: Final = object_value(TypeAdapter(list[JsonValue]).validate_python(created["endpoints"])[0])["id"]
scenario.cleanups.callback(
lambda: gateway.request("DELETE", "/config/pass_through_endpoint", params={"endpoint_id": str(endpoint_id)})
)
return path
def _spend_row(key: str) -> dict[str, JsonValue]:
rows: Final = eventually(
lambda: read_rows(
'SELECT model, spend, total_tokens, status FROM "LiteLLM_SpendLogs" WHERE api_key=%s AND call_type=%s',
(sha256(key.encode()).hexdigest(), "pass_through_endpoint"),
),
lambda values: len(values) == 1,
seconds=70,
)
return rows[0]
def _row_values(row: Mapping[str, JsonValue]) -> tuple[JsonValue, float, JsonValue, JsonValue]:
spend: Final = row["spend"]
assert isinstance(spend, (int, float)), row
return row["model"], float(spend), row["total_tokens"], row["status"]
def test_upstream_reported_model_names_the_spend_row_daily_rollup_and_usage_breakdown(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(_upstream(marker, {**_USAGE_HEADERS, **_MODEL_HEADER})) as wire, gateway.scenario() as scenario:
path: Final = _endpoint(gateway, scenario, marker, wire.url + "/v1/summarize")
team: Final = scenario.team()
key: Final = scenario.key(team_id=team, allowed_passthrough_routes=[path])
response: Final = gateway.request("POST", path, {"ticket_id": marker}, key=key)
assert response.status_code == 200, response.text
assert response.json() == {"summary": marker}, response.text
assert response.headers["x-litellm-upstream-model"] == _REPORTED_MODEL, response.headers
assert _row_values(_spend_row(key)) == (
_REPORTED_MODEL,
pytest.approx(_REPORTED_COST),
_REPORTED_TOKENS,
"success",
)
daily: Final = eventually(
lambda: read_rows('SELECT model, spend FROM "LiteLLM_DailyTeamSpend" WHERE team_id=%s', (team,)),
lambda values: len(values) == 1,
seconds=70,
)
assert (daily[0]["model"], daily[0]["spend"]) == (_REPORTED_MODEL, pytest.approx(_REPORTED_COST)), daily
today: Final = datetime.now(timezone.utc)
activity: Final = gateway.request(
"GET",
"/team/daily/activity",
params={
"team_ids": team,
"start_date": (today - timedelta(days=1)).strftime("%Y-%m-%d"),
"end_date": (today + timedelta(days=1)).strftime("%Y-%m-%d"),
},
)
assert activity.status_code == 200, activity.text
results: Final = object_value(activity.json())["results"]
assert isinstance(results, list) and len(results) == 1, activity.text
models: Final = object_value(object_value(object_value(results[0])["breakdown"])["models"])
assert set(models) == {_REPORTED_MODEL}, activity.text
assert object_value(object_value(models[_REPORTED_MODEL])["metrics"])["spend"] == pytest.approx(
_REPORTED_COST
), activity.text
@pytest.mark.parametrize("stream", [False, True], ids=["buffered", "streamed"])
def test_upstream_reported_model_wins_over_the_request_body_model(gateway: Gateway, stream: bool) -> None:
marker: Final = uuid.uuid4().hex
with (
wire_server(_upstream(marker, {**_USAGE_HEADERS, **_MODEL_HEADER}, stream=stream)) as wire,
gateway.scenario() as scenario,
):
path: Final = _endpoint(gateway, scenario, marker, wire.url + "/v1/summarize")
key: Final = scenario.key(allowed_passthrough_routes=[path])
response: Final = gateway.request("POST", path, {"ticket_id": marker, "model": "gpt-4o"}, key=key)
assert response.status_code == 200, response.text
assert marker in response.text, response.text
assert _row_values(_spend_row(key)) == (
_REPORTED_MODEL,
pytest.approx(_REPORTED_COST),
_REPORTED_TOKENS,
"success",
)
def test_upstream_error_row_carries_the_reported_model_and_cost(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with (
wire_server(_upstream(marker, {**_USAGE_HEADERS, **_MODEL_HEADER}, status=500)) as wire,
gateway.scenario() as scenario,
):
path: Final = _endpoint(gateway, scenario, marker, wire.url + "/v1/summarize")
key: Final = scenario.key(allowed_passthrough_routes=[path])
response: Final = gateway.request("POST", path, {"ticket_id": marker, "model": "gpt-4o"}, key=key)
assert response.status_code == 500, response.text
assert _row_values(_spend_row(key)) == (
_REPORTED_MODEL,
pytest.approx(_REPORTED_COST),
_REPORTED_TOKENS,
"failure",
)
def test_model_header_alone_keeps_the_configured_cost_per_request(gateway: Gateway) -> None:
"""The name is a label: an upstream that reports only a model does not own
the cost, so the endpoint's flat cost_per_request still applies."""
marker: Final = uuid.uuid4().hex
with wire_server(_upstream(marker, _MODEL_HEADER)) as wire, gateway.scenario() as scenario:
path: Final = _endpoint(gateway, scenario, marker, wire.url + "/v1/summarize", cost_per_request=0.5)
key: Final = scenario.key(allowed_passthrough_routes=[path])
response: Final = gateway.request("POST", path, {"ticket_id": marker}, key=key)
assert response.status_code == 200, response.text
model, spend, _, status = _row_values(_spend_row(key))
assert (model, spend, status) == (_REPORTED_MODEL, pytest.approx(0.5), "success")
@pytest.mark.parametrize(
("body_model", "header_value", "expected_model"),
[(None, None, "unknown"), ("gpt-4o", None, "gpt-4o"), ("gpt-4o", " ", "gpt-4o")],
ids=["no_header_no_body_model", "no_header_body_model", "blank_header_body_model"],
)
def test_without_a_usable_model_header_the_row_keeps_the_request_model(
gateway: Gateway, body_model: str | None, header_value: str | None, expected_model: str
) -> None:
marker: Final = uuid.uuid4().hex
headers: Final = {**_USAGE_HEADERS, **({} if header_value is None else {"x-litellm-upstream-model": header_value})}
body: Final[dict[str, JsonValue]] = {"ticket_id": marker, **({} if body_model is None else {"model": body_model})}
with wire_server(_upstream(marker, headers)) as wire, gateway.scenario() as scenario:
path: Final = _endpoint(gateway, scenario, marker, wire.url + "/v1/summarize")
key: Final = scenario.key(allowed_passthrough_routes=[path])
response: Final = gateway.request("POST", path, body, key=key)
assert response.status_code == 200, response.text
assert _row_values(_spend_row(key)) == (
expected_model,
pytest.approx(_REPORTED_COST),
_REPORTED_TOKENS,
"success",
)
@pytest.mark.parametrize(
("header_value", "expected_model"),
[
(f"{_REPORTED_MODEL}, other-name", _REPORTED_MODEL),
("m" * 5000, "m" * 256),
],
ids=["repeated_header_keeps_first", "long_name_cut_to_256"],
)
@pytest.mark.parametrize("status", [200, 500], ids=["success", "failure"])
def test_spend_row_keeps_a_bounded_first_model_name(
gateway: Gateway, header_value: str, expected_model: str, status: int
) -> None:
marker: Final = uuid.uuid4().hex
headers: Final = {**_USAGE_HEADERS, "x-litellm-upstream-model": header_value}
with wire_server(_upstream(marker, headers, status=status)) as wire, gateway.scenario() as scenario:
path: Final = _endpoint(gateway, scenario, marker, wire.url + "/v1/summarize")
key: Final = scenario.key(allowed_passthrough_routes=[path])
response: Final = gateway.request("POST", path, {"ticket_id": marker}, key=key)
assert response.status_code == status, response.text
assert _row_values(_spend_row(key)) == (
expected_model,
pytest.approx(_REPORTED_COST),
_REPORTED_TOKENS,
"success" if status == 200 else "failure",
)

View file

@ -6726,6 +6726,7 @@ async def _run_upstream_reporting_passthrough(
status_code: int = 200,
cost_per_request: float | None = None,
request_headers: Mapping[str, str] | None = None,
request_body: Mapping[str, object] | None = None,
) -> tuple[list[object], MagicMock]:
"""Drive a generic pass-through against an upstream that reports its own
cost/usage. Returns (recorded standard logging payloads, proxy logging mock)."""
@ -6741,13 +6742,14 @@ async def _run_upstream_reporting_passthrough(
)
try:
with ExitStack() as stack:
mock_proxy_logging, enqueued = _enter_upstream_usage_mocks(stack, {})
mock_proxy_logging, enqueued = _enter_upstream_usage_mocks(stack, dict(request_body or {}))
with _recording_success_callback() as recorder:
await pass_through_request(
request=_relay_client_request(method="POST", headers=request_headers),
target="http://internal-api.test/v1/summarize",
custom_headers={},
user_api_key_dict=UserAPIKeyAuth(api_key="sk-upstream-usage", team_id="team-fil"),
custom_body=dict(request_body) if request_body else None,
cost_per_request=cost_per_request,
)
for coroutine in enqueued:
@ -6846,12 +6848,72 @@ async def test_passthrough_error_response_without_usage_headers_records_no_spend
assert "combined_usage_object" not in request_data
_UPSTREAM_SUMMARIZER_HEADERS: Final = MappingProxyType(
{
"x-litellm-response-cost": "0.0123",
"x-litellm-total-tokens": "4210",
"x-litellm-upstream-model": "support-summarizer",
}
)
@pytest.mark.asyncio
async def test_streaming_passthrough_records_cost_and_tokens_reported_by_upstream():
@pytest.mark.parametrize("request_body", [None, {"model": "gpt-4o"}], ids=["no_body_model", "body_model"])
async def test_passthrough_records_model_reported_by_upstream(request_body: Mapping[str, object] | None):
"""A product endpoint takes no `model`, so without the header its spend lands
under `unknown`. The upstream's name wins over a body `model` too, and the
reported cost is kept even though the name is not in the price map."""
payloads, _ = await _run_upstream_reporting_passthrough(_UPSTREAM_SUMMARIZER_HEADERS, request_body=request_body)
assert len(payloads) == 1
logged: Final = cast(types_utils.StandardLoggingPayload, payloads[0])
assert (logged["model"], logged["response_cost"], logged["total_tokens"]) == (
"support-summarizer",
0.0123,
4210,
), logged
@pytest.mark.asyncio
@pytest.mark.parametrize(
("request_body", "expected_model"),
[(None, "unknown"), ({"model": "gpt-4o"}, "gpt-4o")],
ids=["no_body_model", "body_model"],
)
async def test_passthrough_without_model_header_keeps_request_model(
request_body: Mapping[str, object] | None, expected_model: str
):
payloads, _ = await _run_upstream_reporting_passthrough(
{"x-litellm-response-cost": "0.0123", "x-litellm-total-tokens": "4210"}, request_body=request_body
)
assert len(payloads) == 1
assert cast(types_utils.StandardLoggingPayload, payloads[0])["model"] == expected_model, payloads[0]
@pytest.mark.asyncio
async def test_passthrough_records_upstream_reported_model_on_error_response():
import litellm
_, mock_proxy_logging = await _run_upstream_reporting_passthrough(
_UPSTREAM_SUMMARIZER_HEADERS, status_code=500, request_body={"model": "gpt-4o"}
)
mock_proxy_logging.post_call_failure_hook.assert_awaited_once()
request_data: Final = mock_proxy_logging.post_call_failure_hook.await_args.kwargs["request_data"]
assert (request_data["model"], request_data["response_cost"], request_data["combined_usage_object"]) == (
"support-summarizer",
0.0123,
litellm.Usage(total_tokens=4210),
), request_data
@pytest.mark.asyncio
async def test_streaming_passthrough_records_cost_tokens_and_model_reported_by_upstream():
"""
The totals are reported in the response headers, which are known before the
first byte of the stream, so a streamed pass-through must record the same
spend a buffered one does.
The totals and model name are reported in the response headers, which are
known before the first byte of the stream, so a streamed pass-through must
record the same spend and model a buffered one does.
"""
from fastapi.responses import StreamingResponse
@ -6864,6 +6926,7 @@ async def test_streaming_passthrough_records_cost_and_tokens_reported_by_upstrea
"content-type": "text/event-stream",
"x-litellm-response-cost": "0.00312",
"x-litellm-total-tokens": "4021",
"x-litellm-upstream-model": "support-summarizer",
},
stream=_RecordingUpstreamByteStream((b'data: {"delta": "hi"}\n\n',)),
),
@ -6890,6 +6953,7 @@ async def test_streaming_passthrough_records_cost_and_tokens_reported_by_upstrea
assert len(recorder.payloads) == 1
assert recorder.payloads[0]["response_cost"] == 0.00312
assert recorder.payloads[0]["total_tokens"] == 4021
assert recorder.payloads[0]["model"] == "support-summarizer"
@pytest.mark.asyncio

View file

@ -1,12 +1,16 @@
from typing import Final
import httpx
import pytest
from litellm.constants import MAX_SPEND_LOG_MODEL_NAME_LENGTH
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.proxy.pass_through_endpoints.upstream_usage_headers import (
UpstreamReportedUsage,
apply_upstream_reported_model,
apply_upstream_reported_usage,
has_upstream_reported_usage,
parse_upstream_reported_model,
parse_upstream_reported_usage,
)
from litellm.types.utils import Usage
@ -126,3 +130,84 @@ def test_apply_only_overwrites_what_upstream_reported():
assert logging_obj.model_call_details["response_cost"] == 0.5
assert logging_obj.model_call_details["combined_usage_object"] == Usage(total_tokens=42)
@pytest.mark.parametrize(
("raw_model", "expected"),
[
("support-summarizer", "support-summarizer"),
(" support-summarizer \t", "support-summarizer"),
("", None),
(" ", None),
],
)
def test_parse_reads_reported_model_and_treats_blank_as_absent(raw_model: str, expected: str | None):
assert parse_upstream_reported_model(_headers(**{"x-litellm-upstream-model": raw_model})) == expected
def test_parse_returns_no_model_when_upstream_sends_none():
assert parse_upstream_reported_model(_headers(**{"x-litellm-response-cost": "0.5"})) is None
def test_parse_uses_the_first_value_of_a_repeated_model_header():
headers: Final = httpx.Headers(
[("x-litellm-upstream-model", "support-summarizer"), ("x-litellm-upstream-model", "other")]
)
assert parse_upstream_reported_model(headers) == "support-summarizer"
@pytest.mark.parametrize(
("raw_model", "expected"),
[
("m" * 5000, "m" * MAX_SPEND_LOG_MODEL_NAME_LENGTH),
("m" * MAX_SPEND_LOG_MODEL_NAME_LENGTH, "m" * MAX_SPEND_LOG_MODEL_NAME_LENGTH),
("m" * (MAX_SPEND_LOG_MODEL_NAME_LENGTH - 1) + " tail", "m" * (MAX_SPEND_LOG_MODEL_NAME_LENGTH - 1)),
],
ids=["over_limit", "at_limit", "cut_lands_on_a_space"],
)
def test_parse_cuts_a_long_model_name_to_the_spend_log_limit(raw_model: str, expected: str):
assert parse_upstream_reported_model(_headers(**{"x-litellm-upstream-model": raw_model})) == expected
def _logging_obj_for_model(model: str) -> LiteLLMLoggingObj:
logging_obj: Final = _logging_obj()
logging_obj.update_environment_variables(
model=model, user="unknown", optional_params={}, litellm_params={}, call_type="pass_through_endpoint"
)
return logging_obj
def test_apply_model_replaces_the_request_model():
logging_obj: Final = _logging_obj_for_model("gpt-4o")
reported: Final = apply_upstream_reported_model(
logging_obj=logging_obj,
headers=_headers(**{"x-litellm-upstream-model": "support-summarizer"}),
)
assert reported == "support-summarizer"
assert (logging_obj.model, logging_obj.model_call_details["model"]) == ("support-summarizer", "support-summarizer")
def test_apply_model_leaves_request_model_alone_when_header_is_blank():
logging_obj: Final = _logging_obj_for_model("gpt-4o")
assert (
apply_upstream_reported_model(logging_obj=logging_obj, headers=_headers(**{"x-litellm-upstream-model": " "}))
is None
)
assert (logging_obj.model, logging_obj.model_call_details["model"]) == ("gpt-4o", "gpt-4o")
def test_model_header_alone_does_not_make_the_upstream_own_the_cost():
"""The model name is a label: without a cost or token header the request
must still fall back to cost_per_request, not be recorded at 0."""
logging_obj: Final = _logging_obj()
headers: Final = _headers(**{"x-litellm-upstream-model": "support-summarizer"})
apply_upstream_reported_usage(logging_obj=logging_obj, headers=headers)
apply_upstream_reported_model(logging_obj=logging_obj, headers=headers)
assert has_upstream_reported_usage(logging_obj) is False
assert "response_cost" not in logging_obj.model_call_details