mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
test(newrelic): cover static default_team_settings per-team routing (#38857)
* test(newrelic): cover static default_team_settings per-team routing
The dynamic POST /team/{team_id}/callback path for New Relic is tested, but
the static default_team_settings twin had no regression coverage. Add a test
that drives default_team_settings -> add_team_based_callbacks_from_config and
asserts the resolved trusted vars dispatch to BOTH the per-team metrics logger
(cost/usage) and the trace logger (LLM/agent spans), so a config-file customer
gets the same per-team routing as the API customer.
Also correct the /team/callback docstring: callback_name is a str validated
against the credential-capable callbacks, not a fixed langfuse/langsmith/gcs
Literal, and document the newrelic_api_key / newrelic_region vars.
* chore(ui): sync schema.d.ts with the /team/callback docstring
Regenerate the dashboard OpenAPI types for the add_team_callbacks description
change: callback_name is a validated str (not a langfuse/langsmith/gcs
Literal) and the newrelic_api_key / newrelic_region vars are documented.
* docs(newrelic): note LITELLM_OTEL_V2 prerequisite, trim test comments
Address review: team-scoped New Relic config is rejected with a 400 unless the
proxy runs with LITELLM_OTEL_V2=true, so document that in the /team/callback
endpoint and sync schema.d.ts. Drop the narrative setup comments in the new
test per the repo comment convention; the test name and docstring already say why.
This commit is contained in:
parent
d6cce13308
commit
30f3228510
3 changed files with 62 additions and 2 deletions
|
|
@ -252,7 +252,7 @@ async def add_team_callbacks(
|
|||
Use this if if you want different teams to have different success/failure callbacks
|
||||
|
||||
Parameters:
|
||||
- callback_name (Literal["langfuse", "langsmith", "gcs"], required): The name of the callback to add
|
||||
- callback_name (str, required): The name of the callback to add, e.g. "langfuse", "langsmith", "gcs", "newrelic". The value is validated against the callbacks that support team-scoped credentials
|
||||
- callback_type (Literal["success", "failure", "success_and_failure"], required): The type of callback to add. One of:
|
||||
- "success": Callback for successful LLM calls
|
||||
- "failure": Callback for failed LLM calls
|
||||
|
|
@ -268,6 +268,8 @@ async def add_team_callbacks(
|
|||
- langsmith_api_key: The API key for the Langsmith callback
|
||||
- langsmith_project: The project for the Langsmith callback
|
||||
- langsmith_base_url: The base URL for the Langsmith callback
|
||||
- newrelic_api_key: The ingest license key for the team's New Relic account; routes both LLM/agent traces and cost metrics to that account. Requires the proxy to run with LITELLM_OTEL_V2=true, otherwise this callback is rejected with a 400
|
||||
- newrelic_region: The New Relic region for the team's account ("us" or "eu"), riding the team's own key
|
||||
|
||||
Example curl:
|
||||
```
|
||||
|
|
|
|||
|
|
@ -1317,6 +1317,62 @@ def test_proxy_config_state_post_init_callback_call(monkeypatch):
|
|||
assert config["litellm_settings"]["default_team_settings"][0]["team_id"] == "test"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_team_settings_newrelic_resolves_traces_and_metrics():
|
||||
"""Static `default_team_settings` is the config-file twin of POST /team/callback.
|
||||
|
||||
A team pinned to New Relic through `default_team_settings` must reach the
|
||||
same two loggers the dynamic path does: the per-team metrics logger (cost
|
||||
and usage) and the trace logger (LLM/agent spans). This proves the static
|
||||
path resolves both, not just one, so the config-file customer gets the
|
||||
same per-team routing as the API customer.
|
||||
"""
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
|
||||
from litellm.proxy.proxy_server import ProxyConfig
|
||||
|
||||
pc = ProxyConfig()
|
||||
pc.config = {
|
||||
"litellm_settings": {
|
||||
"default_team_settings": [
|
||||
{
|
||||
"team_id": "team-a",
|
||||
"success_callback": ["newrelic"],
|
||||
"newrelic_api_key": "team-a-ingest-key",
|
||||
"newrelic_region": "eu",
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
callback_metadata = LiteLLMProxyRequestSetup.add_team_based_callbacks_from_config(
|
||||
team_id="team-a",
|
||||
proxy_config=pc,
|
||||
)
|
||||
|
||||
assert callback_metadata is not None
|
||||
assert callback_metadata.success_callback == ["newrelic"]
|
||||
assert callback_metadata.callback_vars == {
|
||||
"newrelic_api_key": "team-a-ingest-key",
|
||||
"newrelic_region": "eu",
|
||||
}
|
||||
|
||||
logging_obj = Logging(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=False,
|
||||
call_type="completion",
|
||||
start_time=None,
|
||||
litellm_call_id="static-nr-1",
|
||||
function_id="static-nr-1",
|
||||
)
|
||||
logging_obj._trusted_callback_vars = tuple(callback_metadata.callback_vars.items())
|
||||
|
||||
resolved = logging_obj._resolve_dynamic_callback_string("newrelic")
|
||||
resolved_names = {type(logger).__name__ for logger in resolved}
|
||||
assert resolved_names == {"NewRelicMetricsLogger", "NewRelicLogger"}
|
||||
|
||||
|
||||
def test_proxy_config_state_get_config_state_error():
|
||||
"""
|
||||
Ensures that get_config_state does not raise an error when the config is not a valid dictionary
|
||||
|
|
|
|||
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -15505,7 +15505,7 @@ export interface paths {
|
|||
* Use this if if you want different teams to have different success/failure callbacks
|
||||
*
|
||||
* Parameters:
|
||||
* - callback_name (Literal["langfuse", "langsmith", "gcs"], required): The name of the callback to add
|
||||
* - callback_name (str, required): The name of the callback to add, e.g. "langfuse", "langsmith", "gcs", "newrelic". The value is validated against the callbacks that support team-scoped credentials
|
||||
* - callback_type (Literal["success", "failure", "success_and_failure"], required): The type of callback to add. One of:
|
||||
* - "success": Callback for successful LLM calls
|
||||
* - "failure": Callback for failed LLM calls
|
||||
|
|
@ -15521,6 +15521,8 @@ export interface paths {
|
|||
* - langsmith_api_key: The API key for the Langsmith callback
|
||||
* - langsmith_project: The project for the Langsmith callback
|
||||
* - langsmith_base_url: The base URL for the Langsmith callback
|
||||
* - newrelic_api_key: The ingest license key for the team's New Relic account; routes both LLM/agent traces and cost metrics to that account. Requires the proxy to run with LITELLM_OTEL_V2=true, otherwise this callback is rejected with a 400
|
||||
* - newrelic_region: The New Relic region for the team's account ("us" or "eu"), riding the team's own key
|
||||
*
|
||||
* Example curl:
|
||||
* ```
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue