diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 152c27202b4..8fab485cc7a 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -27,7 +27,16 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars +from litellm.proxy.common_utils.callback_utils import ( + _CALLBACK_VAR_ENCRYPTED_PREFIX, + decrypt_callback_vars, + encrypt_callback_vars, + is_sensitive_callback_key, +) +from litellm.proxy.litellm_pre_call_utils import ( + _get_validated_callback_metadata, + convert_key_logging_metadata_to_callback, +) from litellm.proxy.management_endpoints.team_endpoints import _verify_team_access from litellm.proxy.management_helpers.utils import management_endpoint_wrapper from litellm.repositories.team_repository import TeamRepository @@ -64,6 +73,76 @@ def _redact_callback_secrets(metadata: Any) -> Any: return redacted +def _mask_sensitive_callback_vars(callbacks: TeamCallbackMetadata) -> None: + """Mask credential-bearing callback vars in place, keeping the rest readable. + + ``callback_vars`` mixes credentials (``langsmith_api_key``, + ``langfuse_secret_key``, ``gcs_path_service_account``) with plain + configuration (project names, bucket names, hosts). The configuration is + what makes a read of this endpoint useful, so only the sensitive keys are + replaced, using the same marker as the audit-log redaction above. + + A value that still carries the encrypted prefix here failed to decrypt, so + it is masked too. Handing back a ciphertext blob under a key that is not + classified as sensitive would give the caller something it cannot use and + cannot tell apart from a real value. + + Masking in place rather than rebuilding the mapping keeps this under the + LIT002 mutable-collection-construction budget. It is safe because the only + caller passes an object it just built from a decrypted deep copy of the + row, so nothing here is reachable from the team's stored metadata. + """ + if not callbacks.callback_vars: + return + for key in tuple(callbacks.callback_vars): + value = callbacks.callback_vars[key] + if is_sensitive_callback_key(key) or str(value).startswith(_CALLBACK_VAR_ENCRYPTED_PREFIX): + callbacks.callback_vars[key] = _CALLBACK_VARS_REDACTED + + +def _resolve_team_callbacks(team_metadata: object) -> TeamCallbackMetadata: + """Report the callbacks that are actually in effect for a team. + + A team's callback config can live in either of two metadata slots. + ``metadata["logging"]`` holds the ``AddTeamCallback`` entries written by + ``POST /team/{team_id}/callback`` and by the Admin UI, while + ``metadata["callback_settings"]`` holds the older ``TeamCallbackMetadata`` + shape. Request-time resolution in ``_get_dynamic_logging_metadata`` treats + the two as mutually exclusive: a populated ``logging`` slot wins outright + and ``callback_settings`` is consulted only as the deprecated fallback. + This reader applies the same precedence so it reports what a request would + really do. Merging the two instead would report a ``callback_settings`` + entry as active for a team whose requests never fire it. + + Credential ``callback_vars`` are stored encrypted, so they are decrypted + before being masked by key; a value encrypted under a key that is no longer + classified as sensitive would otherwise come back as raw ciphertext. + """ + if not isinstance(team_metadata, dict): + return TeamCallbackMetadata() + + decrypted = decrypt_callback_vars(team_metadata) + logging_entries = decrypted.get("logging") + + if logging_entries is not None: + resolved = TeamCallbackMetadata() + for entry in logging_entries if isinstance(logging_entries, list) else (): + if not isinstance(entry, dict): + continue + callback = _get_validated_callback_metadata(item=entry, source="team-level read") + if callback is None: + continue + resolved = convert_key_logging_metadata_to_callback(data=callback, team_callback_settings_obj=resolved) + else: + callback_settings = decrypted.get("callback_settings") + resolved = ( + TeamCallbackMetadata(**callback_settings) if isinstance(callback_settings, dict) else TeamCallbackMetadata() + ) + + _mask_sensitive_callback_vars(resolved) + return resolved + + def _log_audit_task_exception(task: "asyncio.Task[None]") -> None: """Surface a fire-and-forget audit-log task failure. @@ -408,6 +487,12 @@ async def get_team_callbacks( This will return the callback settings for the team with id dbe2f686-a686-4896-864a-4c3924458709 + Covers callbacks registered through POST /team/{team_id}/callback and the Admin UI as well as + teams still on the deprecated callback_settings shape, resolved from the team's stored metadata + with the same precedence used at request time. A key-level logging config overrides the team's + at request time and is not reflected here. Credential-bearing callback_vars are returned masked + as `***REDACTED***` + Returns { "status": "success", "data": { @@ -440,12 +525,7 @@ async def get_team_callbacks( user_api_key_dict=user_api_key_dict, ) - # Retrieve team callback settings from metadata - team_metadata = _existing_team.metadata - team_callback_settings = team_metadata.get("callback_settings", {}) - - # Convert to TeamCallbackMetadata object for consistent structure - team_callback_settings_obj = TeamCallbackMetadata(**team_callback_settings) + team_callback_settings_obj = _resolve_team_callbacks(_existing_team.metadata) return { "status": "success", diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py index d43bf3a3bd8..3b2b1ccb793 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py @@ -459,3 +459,262 @@ async def test_add_team_callbacks_writes_encrypted_callback_vars(monkeypatch): recovered = decrypt_callback_vars(written)["logging"][0]["callback_vars"] assert recovered["langfuse_secret_key"] == "sk-lf-real-secret" assert recovered["langfuse_public_key"] == "pk-lf-real-public" + + +@pytest.mark.asyncio +async def test_get_team_callbacks_returns_callbacks_registered_via_post(monkeypatch): + """POST then GET must round-trip. + + add_team_callbacks writes metadata["logging"]; a GET that reads only + metadata["callback_settings"] reports an empty list for every team + configured through the API. The row handed to the GET here is the exact + payload the POST wrote, so the two code paths cannot drift apart again. + """ + monkeypatch.setenv("LITELLM_SALT_KEY", "test-salt-32-bytes-aaaaaaaaaaaaaa") + row = _team_row(team_id="team-1", metadata={}) + mock_prisma = _patch_prisma(row) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin"), + patch("litellm.proxy.proxy_server.master_key", None), + ): + await add_team_callbacks( + data=AddTeamCallback( + callback_name="langsmith", + callback_type="success", + callback_vars={ + "langsmith_api_key": "lsv2-real-secret", + "langsmith_project": "tenant-project", + }, + ), + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + + # Feed the GET exactly what the POST persisted. + row.metadata = json.loads(mock_prisma.db.litellm_teamtable.update.await_args.kwargs["data"]["metadata"]) + row.model_dump.return_value["metadata"] = row.metadata + + response = await get_team_callbacks( + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + ) + + assert response["data"]["success_callbacks"] == ["langsmith"] + assert response["data"]["failure_callbacks"] == [] + # Non-secret vars come back usable, the credential is masked, and the + # ciphertext that is stored on the row never reaches the response. + assert response["data"]["callback_vars"]["langsmith_project"] == "tenant-project" + assert response["data"]["callback_vars"]["langsmith_api_key"] == "***REDACTED***" + assert "lsv2-real-secret" not in json.dumps(response) + assert "litellm_enc::" not in json.dumps(response) + + +@pytest.mark.asyncio +async def test_get_team_callbacks_prefers_logging_over_deprecated_callback_settings(): + """A team carrying both shapes must report only the one that actually fires. + + Request-time resolution in _get_dynamic_logging_metadata stops at the + first populated slot: metadata["logging"] wins and callback_settings is + never consulted. Reporting the union here would tell an operator that + gcs_bucket is active on a team whose requests never send to it. + """ + metadata = { + "callback_settings": { + "success_callback": ["gcs_bucket"], + "failure_callback": [], + "callback_vars": {"gcs_bucket_name": "legacy-bucket"}, + }, + "logging": [ + { + "callback_name": "langsmith", + "callback_type": "success_and_failure", + "callback_vars": {"langsmith_project": "tenant-project"}, + }, + {"callback_name": "missing-required-fields"}, + ], + } + mock_prisma = _patch_prisma(_team_row(team_id="team-1", metadata=metadata)) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma): + response = await get_team_callbacks( + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + ) + + assert response["data"]["success_callbacks"] == ["langsmith"] + assert response["data"]["failure_callbacks"] == ["langsmith"] + # The deprecated slot contributes nothing, and the malformed logging entry + # is skipped rather than failing the whole read. + assert response["data"]["callback_vars"] == {"langsmith_project": "tenant-project"} + + +@pytest.mark.asyncio +async def test_get_team_callbacks_reports_nothing_when_logging_slot_is_empty(): + """An empty logging slot must not fall through to callback_settings. + + Request-time resolution stops at the first non-None slot, so a team whose + logging list is empty fires no callbacks at all even when the deprecated + shape is still populated next to it. Reporting gcs_bucket here would tell + an operator a destination is live when nothing is being sent to it. + """ + metadata = { + "logging": [], + "callback_settings": { + "success_callback": ["gcs_bucket"], + "failure_callback": [], + "callback_vars": {"gcs_bucket_name": "legacy-bucket"}, + }, + } + mock_prisma = _patch_prisma(_team_row(team_id="team-1", metadata=metadata)) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma): + response = await get_team_callbacks( + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + ) + + assert response["data"]["success_callbacks"] == [] + assert response["data"]["failure_callbacks"] == [] + assert response["data"]["callback_vars"] == {} + + +@pytest.mark.asyncio +async def test_get_team_callbacks_decrypts_vars_stored_under_non_sensitive_keys(monkeypatch): + """Stored ciphertext must be decrypted, not handed back raw. + + Which keys count as sensitive is a moving classification, so a value can be + encrypted at rest under a key that later stops being masked on read. Without + the decrypt step that value comes back as an unusable litellm_enc:: blob. + """ + from litellm.proxy.common_utils.callback_utils import _CALLBACK_VAR_ENCRYPTED_PREFIX, is_sensitive_callback_key + from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper + + monkeypatch.setenv("LITELLM_SALT_KEY", "test-salt-32-bytes-aaaaaaaaaaaaaa") + assert not is_sensitive_callback_key("langsmith_project"), "test needs a key that is not masked on read" + + metadata = { + "logging": [ + { + "callback_name": "langsmith", + "callback_type": "success", + "callback_vars": { + "langsmith_project": _CALLBACK_VAR_ENCRYPTED_PREFIX + encrypt_value_helper("tenant-project"), + }, + } + ] + } + mock_prisma = _patch_prisma(_team_row(team_id="team-1", metadata=metadata)) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.master_key", None), + ): + response = await get_team_callbacks( + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + ) + + assert response["data"]["callback_vars"]["langsmith_project"] == "tenant-project" + + +@pytest.mark.asyncio +async def test_get_team_callbacks_masks_values_that_fail_to_decrypt(monkeypatch): + """A value that cannot be decrypted must never leave as ciphertext. + + After a salt-key rotation an existing value no longer decrypts, and the + shared helper passes it through untouched. Under a key that is not + classified as sensitive it would otherwise reach the caller as an opaque + blob that is indistinguishable from a real value. + """ + from litellm.proxy.common_utils.callback_utils import _CALLBACK_VAR_ENCRYPTED_PREFIX + from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper + + monkeypatch.setenv("LITELLM_SALT_KEY", "test-salt-32-bytes-aaaaaaaaaaaaaa") + stale = _CALLBACK_VAR_ENCRYPTED_PREFIX + encrypt_value_helper("tenant-project") + monkeypatch.setenv("LITELLM_SALT_KEY", "test-salt-32-bytes-bbbbbbbbbbbbbb") + + metadata = { + "logging": [ + { + "callback_name": "langsmith", + "callback_type": "success", + "callback_vars": {"langsmith_project": stale}, + } + ] + } + mock_prisma = _patch_prisma(_team_row(team_id="team-1", metadata=metadata)) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.master_key", None), + ): + response = await get_team_callbacks( + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + ) + + assert response["data"]["success_callbacks"] == ["langsmith"] + assert response["data"]["callback_vars"]["langsmith_project"] == "***REDACTED***" + assert _CALLBACK_VAR_ENCRYPTED_PREFIX not in json.dumps(response) + + +@pytest.mark.asyncio +async def test_get_team_callbacks_falls_back_to_deprecated_callback_settings(): + """Teams that never used the API keep working: with no logging slot, the + deprecated callback_settings shape is still reported, exactly as the + request-time fallback would use it.""" + metadata = { + "callback_settings": { + "success_callback": ["gcs_bucket"], + "failure_callback": ["langfuse"], + "callback_vars": { + "gcs_bucket_name": "legacy-bucket", + "langfuse_secret_key": "sk-lf-legacy-plaintext", + }, + } + } + mock_prisma = _patch_prisma(_team_row(team_id="team-1", metadata=metadata)) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma): + response = await get_team_callbacks( + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + ) + + assert response["data"]["success_callbacks"] == ["gcs_bucket"] + assert response["data"]["failure_callbacks"] == ["langfuse"] + assert response["data"]["callback_vars"]["gcs_bucket_name"] == "legacy-bucket" + # Legacy rows predate encryption at rest, so this endpoint is where the + # plaintext secret would otherwise escape. + assert response["data"]["callback_vars"]["langfuse_secret_key"] == "***REDACTED***" + assert "sk-lf-legacy-plaintext" not in json.dumps(response) + + +@pytest.mark.asyncio +async def test_get_team_callbacks_reports_empty_for_team_without_callbacks(): + """A team with no callback config still returns the documented empty shape.""" + mock_prisma = _patch_prisma(_team_row(team_id="team-1", metadata={})) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma): + response = await get_team_callbacks( + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + ) + + assert response["data"] == { + "team_id": "team-1", + "success_callbacks": [], + "failure_callbacks": [], + "callback_vars": {}, + } diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 90552b6eae1..d24937fd672 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -14019,6 +14019,12 @@ export interface paths { * * This will return the callback settings for the team with id dbe2f686-a686-4896-864a-4c3924458709 * + * Covers callbacks registered through POST /team/{team_id}/callback and the Admin UI as well as + * teams still on the deprecated callback_settings shape, resolved from the team's stored metadata + * with the same precedence used at request time. A key-level logging config overrides the team's + * at request time and is not reflected here. Credential-bearing callback_vars are returned masked + * as `***REDACTED***` + * * Returns { * "status": "success", * "data": {