mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-08 03:08:45 +00:00
Remove unneeded metadata info from LangSmith
Some checks failed
Unit Tests: Caching (Redis) / caching-redis (push) Has been cancelled
Unit Tests: Proxy DB Operations / assert-shard-coverage (push) Has been cancelled
Unit Tests: Security / security (push) Has been cancelled
Unit Tests: Proxy DB Operations / proxy-utils (push) Has been cancelled
Unit Tests: Proxy DB Operations / auth-checks (push) Has been cancelled
Unit Tests: Proxy DB Operations / budgets (push) Has been cancelled
Unit Tests: Proxy DB Operations / custom-logging (push) Has been cancelled
Unit Tests: Proxy DB Operations / db-and-spend (push) Has been cancelled
Unit Tests: Proxy DB Operations / endpoints-and-responses (push) Has been cancelled
Unit Tests: Proxy DB Operations / guardrails-hooks (push) Has been cancelled
Unit Tests: Proxy DB Operations / jwt-and-keys (push) Has been cancelled
Unit Tests: Proxy DB Operations / key-generation (push) Has been cancelled
Unit Tests: Proxy DB Operations / logging-misc (push) Has been cancelled
Unit Tests: Proxy DB Operations / proxy-runtime (push) Has been cancelled
Unit Tests: Proxy DB Operations / proxy-server-core (push) Has been cancelled
Unit Tests: Proxy DB Operations / schema-migration (push) Has been cancelled
Some checks failed
Unit Tests: Caching (Redis) / caching-redis (push) Has been cancelled
Unit Tests: Proxy DB Operations / assert-shard-coverage (push) Has been cancelled
Unit Tests: Security / security (push) Has been cancelled
Unit Tests: Proxy DB Operations / proxy-utils (push) Has been cancelled
Unit Tests: Proxy DB Operations / auth-checks (push) Has been cancelled
Unit Tests: Proxy DB Operations / budgets (push) Has been cancelled
Unit Tests: Proxy DB Operations / custom-logging (push) Has been cancelled
Unit Tests: Proxy DB Operations / db-and-spend (push) Has been cancelled
Unit Tests: Proxy DB Operations / endpoints-and-responses (push) Has been cancelled
Unit Tests: Proxy DB Operations / guardrails-hooks (push) Has been cancelled
Unit Tests: Proxy DB Operations / jwt-and-keys (push) Has been cancelled
Unit Tests: Proxy DB Operations / key-generation (push) Has been cancelled
Unit Tests: Proxy DB Operations / logging-misc (push) Has been cancelled
Unit Tests: Proxy DB Operations / proxy-runtime (push) Has been cancelled
Unit Tests: Proxy DB Operations / proxy-server-core (push) Has been cancelled
Unit Tests: Proxy DB Operations / schema-migration (push) Has been cancelled
This commit is contained in:
parent
0b87bdff3d
commit
214dea43fa
2 changed files with 94 additions and 0 deletions
|
|
@ -19,6 +19,7 @@ from litellm.integrations.langsmith_mock_client import (
|
|||
create_mock_langsmith_client,
|
||||
should_use_langsmith_mock,
|
||||
)
|
||||
from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
get_async_httpx_client,
|
||||
httpxSpecialProvider,
|
||||
|
|
@ -153,6 +154,15 @@ class LangsmithLogger(CustomBatchLogger):
|
|||
for key in ("session_id", "thread_id", "conversation_id"):
|
||||
if key in requester_metadata and key not in extra_metadata:
|
||||
extra_metadata[key] = requester_metadata[key]
|
||||
|
||||
# helper is shallow; also scrub nested requester_metadata since
|
||||
# LangSmith forwards the whole dict into `extra`
|
||||
extra_metadata = redact_user_api_key_info(metadata=extra_metadata)
|
||||
nested = extra_metadata.get("requester_metadata")
|
||||
if isinstance(nested, dict):
|
||||
extra_metadata["requester_metadata"] = redact_user_api_key_info(
|
||||
metadata=nested
|
||||
)
|
||||
return extra_metadata
|
||||
|
||||
def _build_outputs_with_usage(
|
||||
|
|
|
|||
|
|
@ -6,9 +6,18 @@ import pytest
|
|||
|
||||
sys.path.insert(0, os.path.abspath("../.."))
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.langsmith import LangsmithLogger
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def reset_redact_flag():
|
||||
"""Reset redact_user_api_key_info between tests so global state doesn't leak."""
|
||||
original = litellm.redact_user_api_key_info
|
||||
yield
|
||||
litellm.redact_user_api_key_info = original
|
||||
|
||||
|
||||
class TestLangsmithLoggerInit:
|
||||
"""Test cases for LangSmith logger initialization, particularly sampling rate handling.
|
||||
|
||||
|
|
@ -263,3 +272,78 @@ class TestLangsmithPrepareLogData:
|
|||
assert um["input_tokens"] == 100
|
||||
assert um["output_tokens"] == 50
|
||||
assert um["total_tokens"] == 150
|
||||
|
||||
|
||||
class TestLangsmithRedactUserApiKeyInfo:
|
||||
"""Verify litellm.redact_user_api_key_info is honored for LangSmith."""
|
||||
|
||||
def _logger(self):
|
||||
return LangsmithLogger(
|
||||
langsmith_api_key="test-key",
|
||||
langsmith_project="test-project",
|
||||
)
|
||||
|
||||
def _metadata_with_user_api_key_fields(self):
|
||||
return {
|
||||
"user_api_key_hash": "abc123",
|
||||
"user_api_key_alias": "engineer-key",
|
||||
"user_api_key_user_id": "default_user_id",
|
||||
"user_api_key_team_id": "team-uuid",
|
||||
"user_api_key_team_alias": "GNT",
|
||||
"user_api_key_request_route": "/chat/completions",
|
||||
"user_api_key_spend": 1.64,
|
||||
"model": "gpt-4",
|
||||
"requester_metadata": {
|
||||
"user_api_key_team_id": "team-uuid",
|
||||
"user_api_key_user_id": "default_user_id",
|
||||
"session_id": "sess-1",
|
||||
},
|
||||
}
|
||||
|
||||
def test_redact_disabled_keeps_user_api_key_fields(self, reset_redact_flag):
|
||||
"""Flag off: user_api_key_* fields are preserved (no behavior change)."""
|
||||
litellm.redact_user_api_key_info = False
|
||||
logger = self._logger()
|
||||
metadata = self._metadata_with_user_api_key_fields()
|
||||
|
||||
extra = logger._build_extra_metadata(metadata)
|
||||
|
||||
assert extra["user_api_key_hash"] == "abc123"
|
||||
assert extra["user_api_key_team_id"] == "team-uuid"
|
||||
assert extra["requester_metadata"]["user_api_key_user_id"] == "default_user_id"
|
||||
|
||||
def test_redact_enabled_strips_top_level_user_api_key_fields(
|
||||
self, reset_redact_flag
|
||||
):
|
||||
"""Flag on: top-level user_api_key_* keys removed; other keys preserved."""
|
||||
litellm.redact_user_api_key_info = True
|
||||
logger = self._logger()
|
||||
metadata = self._metadata_with_user_api_key_fields()
|
||||
|
||||
extra = logger._build_extra_metadata(metadata)
|
||||
|
||||
for key in (
|
||||
"user_api_key_hash",
|
||||
"user_api_key_alias",
|
||||
"user_api_key_user_id",
|
||||
"user_api_key_team_id",
|
||||
"user_api_key_team_alias",
|
||||
"user_api_key_request_route",
|
||||
"user_api_key_spend",
|
||||
):
|
||||
assert key not in extra, f"{key} should be redacted at top level"
|
||||
assert extra["model"] == "gpt-4"
|
||||
|
||||
def test_redact_enabled_strips_nested_requester_metadata(self, reset_redact_flag):
|
||||
"""Flag on: nested requester_metadata.user_api_key_* removed; session_id still lifted."""
|
||||
litellm.redact_user_api_key_info = True
|
||||
logger = self._logger()
|
||||
metadata = self._metadata_with_user_api_key_fields()
|
||||
|
||||
extra = logger._build_extra_metadata(metadata)
|
||||
|
||||
nested = extra["requester_metadata"]
|
||||
assert "user_api_key_team_id" not in nested
|
||||
assert "user_api_key_user_id" not in nested
|
||||
assert nested["session_id"] == "sess-1"
|
||||
assert extra["session_id"] == "sess-1"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue