From 0cfe667a89b687db2a8d1c2772ae615553670f57 Mon Sep 17 00:00:00 2001 From: Quinn Xu Date: Thu, 17 Sep 2026 18:38:28 +0000 Subject: [PATCH 1/7] fix(datadog): set top-level model_name on embedding LLMObs spans Datadog prices llm/embedding spans from meta.model_name and meta.model_provider. LiteLLM only put those under meta.metadata, so embedding spans were flagged Partial cost. Lift the existing values onto Meta for llm and embedding kinds. Part of #41601 --- .../integrations/datadog/datadog_llm_obs.py | 6 ++++ litellm/types/integrations/datadog_llm_obs.py | 4 +++ .../datadog/test_datadog_llm_obs.py | 34 +++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index c64a12c6d75..1e6902ca852 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -654,6 +654,12 @@ class DataDogLLMObsLogger(CustomBatchLogger): "error": error_info, **({"tool_definitions": tool_definitions} if tool_definitions else {}), } + # Datadog prices llm/embedding spans from top-level meta.model_name / + # model_provider. Those values already live in metadata; lift them so + # embedding spans are not flagged "Partial cost — Unsupported model provider". + if span_kind in ("llm", "embedding"): + meta["model_name"] = standard_logging_payload.get("model", "unknown") + meta["model_provider"] = standard_logging_payload.get("custom_llm_provider", "unknown") metrics: Final = self._assemble_metrics(standard_logging_payload, tool_output_tokens) diff --git a/litellm/types/integrations/datadog_llm_obs.py b/litellm/types/integrations/datadog_llm_obs.py index f4fcabf53ed..1e387266df2 100644 --- a/litellm/types/integrations/datadog_llm_obs.py +++ b/litellm/types/integrations/datadog_llm_obs.py @@ -69,6 +69,10 @@ class DDLLMObsError(TypedDict, total=False): class Meta(TypedDict, total=False): # The span kind: "agent", "workflow", "llm", "tool", "task", "embedding", or "retrieval". kind: Literal["llm", "tool", "task", "embedding", "retrieval"] + # Top-level model fields required for Datadog cost attribution on llm/embedding spans. + # Values are also kept under metadata for backwards compatibility. + model_name: str + model_provider: str input: InputMeta # The span's input information. output: OutputMeta # The span's output information. metadata: dict[str, Any] diff --git a/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py b/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py index a2f81091893..7cb8f16e1fe 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_llm_obs.py @@ -1122,3 +1122,37 @@ def test_reasoning_content_survives_the_mapping(logger: DataDogLLMObsLogger) -> ) assert payload["meta"]["output"]["messages"][0]["reasoning_content"] == "thinking" + + +def test_embedding_span_sets_top_level_model_name(logger: DataDogLLMObsLogger) -> None: + """Datadog prices embedding spans from meta.model_name, not meta.metadata (issue #41601).""" + kwargs = build_payload(messages=None) + kwargs["standard_logging_object"]["call_type"] = "aembedding" + kwargs["standard_logging_object"]["model"] = "openai/text-embedding-3-small" + kwargs["standard_logging_object"]["custom_llm_provider"] = "openai" + kwargs["standard_logging_object"]["response"] = {"data": [{"embedding": [0.1, 0.2]}]} + kwargs["litellm_params"]["metadata"]["parent_id"] = "parent-span-1" + + start = datetime(2026, 9, 1, 12, 0, 0) + span = json.loads(safe_dumps(logger.create_llm_obs_payload(kwargs, start, start + timedelta(seconds=1)))) + + assert span["meta"]["kind"] == "embedding" + assert span["meta"]["model_name"] == "openai/text-embedding-3-small" + assert span["meta"]["model_provider"] == "openai" + # Still present under metadata for backwards compatibility. + assert span["meta"]["metadata"]["model_name"] == "openai/text-embedding-3-small" + assert span["meta"]["metadata"]["model_provider"] == "openai" + + +def test_llm_span_sets_top_level_model_name(logger: DataDogLLMObsLogger) -> None: + """llm spans must keep top-level model_name/model_provider for Datadog cost attribution.""" + kwargs = build_payload() + kwargs["standard_logging_object"]["model"] = "openai/gpt-4.1-mini" + kwargs["standard_logging_object"]["custom_llm_provider"] = "openai" + + start = datetime(2026, 9, 1, 12, 0, 0) + span = json.loads(safe_dumps(logger.create_llm_obs_payload(kwargs, start, start + timedelta(seconds=1)))) + + assert span["meta"]["kind"] == "llm" + assert span["meta"]["model_name"] == "openai/gpt-4.1-mini" + assert span["meta"]["model_provider"] == "openai" From 849d7f91f460cda3146bcaf448eaec5f058897fb Mon Sep 17 00:00:00 2001 From: Quinn Xu Date: Sun, 11 Oct 2026 10:02:36 -0700 Subject: [PATCH 2/7] fix(datadog): mark Meta model_name/provider ReadOnly for LIT012 TypedDict writable fields trip the type-discipline gate; match neighboring Meta fields by wrapping the new keys in ReadOnly. --- litellm/types/integrations/datadog_llm_obs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/types/integrations/datadog_llm_obs.py b/litellm/types/integrations/datadog_llm_obs.py index 1e387266df2..38c5d6adffd 100644 --- a/litellm/types/integrations/datadog_llm_obs.py +++ b/litellm/types/integrations/datadog_llm_obs.py @@ -71,8 +71,8 @@ class Meta(TypedDict, total=False): kind: Literal["llm", "tool", "task", "embedding", "retrieval"] # Top-level model fields required for Datadog cost attribution on llm/embedding spans. # Values are also kept under metadata for backwards compatibility. - model_name: str - model_provider: str + model_name: ReadOnly[str] + model_provider: ReadOnly[str] input: InputMeta # The span's input information. output: OutputMeta # The span's output information. metadata: dict[str, Any] From df0d1c36b022e102eacd9179505f0d800bb717dc Mon Sep 17 00:00:00 2001 From: Quinn Xu Date: Sun, 11 Oct 2026 10:12:06 -0700 Subject: [PATCH 3/7] fix(datadog): build Meta model identity without TypedDict mutation Include model_name/model_provider in the Meta literal for llm/embedding spans instead of assigning NotRequired ReadOnly keys after construction, which grew the basedpyright TypedDict/unknown-member ceilings. --- .../integrations/datadog/datadog_llm_obs.py | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index 4dbdf589253..bdf51654c7c 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -657,6 +657,19 @@ class DataDogLLMObsLogger(CustomBatchLogger): redact_prompt_text=redact_payload, ) + # Datadog prices llm/embedding spans from top-level meta.model_name / + # model_provider. Those values already live in metadata; lift them so + # embedding spans are not flagged "Partial cost — Unsupported model provider". + model_identity: Final[Meta] = ( + { + "model_name": str(standard_logging_payload.get("model") or "unknown"), + "model_provider": str( + standard_logging_payload.get("custom_llm_provider") or "unknown" + ), + } + if span_kind in ("llm", "embedding") + else {} + ) meta: Final[Meta] = { "kind": span_kind, "input": input_meta, @@ -664,13 +677,8 @@ class DataDogLLMObsLogger(CustomBatchLogger): "metadata": payload_metadata, "error": error_info, **({"tool_definitions": tool_definitions} if tool_definitions else {}), + **model_identity, } - # Datadog prices llm/embedding spans from top-level meta.model_name / - # model_provider. Those values already live in metadata; lift them so - # embedding spans are not flagged "Partial cost — Unsupported model provider". - if span_kind in ("llm", "embedding"): - meta["model_name"] = standard_logging_payload.get("model", "unknown") - meta["model_provider"] = standard_logging_payload.get("custom_llm_provider", "unknown") metrics: Final = self._assemble_metrics(standard_logging_payload, tool_output_tokens) From 4ad7dc6a76be98c917176270221eaa0fbfde9907 Mon Sep 17 00:00:00 2001 From: Quinn Xu Date: Sun, 11 Oct 2026 10:16:20 -0700 Subject: [PATCH 4/7] style(datadog): ruff-format Meta model_identity construction --- litellm/integrations/datadog/datadog_llm_obs.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index bdf51654c7c..9ede440c9dc 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -663,9 +663,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): model_identity: Final[Meta] = ( { "model_name": str(standard_logging_payload.get("model") or "unknown"), - "model_provider": str( - standard_logging_payload.get("custom_llm_provider") or "unknown" - ), + "model_provider": str(standard_logging_payload.get("custom_llm_provider") or "unknown"), } if span_kind in ("llm", "embedding") else {} From 713a1a64b2dff29529d4af67aca4aef779a8ea78 Mon Sep 17 00:00:00 2001 From: Quinn Xu Date: Tue, 13 Oct 2026 10:13:54 -0700 Subject: [PATCH 5/7] fix(datadog): type-safe top-level model_name on LLMObs Meta Use required TypedDict key access for model / custom_llm_provider instead of str(.get(...)), so the basedpyright unknown-argument/member ceilings do not grow on this change. --- .../integrations/datadog/datadog_llm_obs.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index 9ede440c9dc..4dcb40c7f8f 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -660,14 +660,11 @@ class DataDogLLMObsLogger(CustomBatchLogger): # Datadog prices llm/embedding spans from top-level meta.model_name / # model_provider. Those values already live in metadata; lift them so # embedding spans are not flagged "Partial cost — Unsupported model provider". - model_identity: Final[Meta] = ( - { - "model_name": str(standard_logging_payload.get("model") or "unknown"), - "model_provider": str(standard_logging_payload.get("custom_llm_provider") or "unknown"), - } - if span_kind in ("llm", "embedding") - else {} - ) + # Use TypedDict required-key access (not str(.get(...))) so basedpyright + # reportUnknownArgumentType / reportUnknownMemberType ceilings stay flat. + model_name: Final[str] = standard_logging_payload["model"] or "unknown" + provider_raw: Final[str | None] = standard_logging_payload["custom_llm_provider"] + model_provider: Final[str] = provider_raw if provider_raw is not None else "unknown" meta: Final[Meta] = { "kind": span_kind, "input": input_meta, @@ -675,7 +672,11 @@ class DataDogLLMObsLogger(CustomBatchLogger): "metadata": payload_metadata, "error": error_info, **({"tool_definitions": tool_definitions} if tool_definitions else {}), - **model_identity, + **( + {"model_name": model_name, "model_provider": model_provider} + if span_kind in ("llm", "embedding") + else {} + ), } metrics: Final = self._assemble_metrics(standard_logging_payload, tool_output_tokens) From 4826e9bba29b1faccee8a3bd2f25c3d00968d7ec Mon Sep 17 00:00:00 2001 From: Quinn Xu Date: Thu, 15 Oct 2026 10:00:05 -0700 Subject: [PATCH 6/7] fix(datadog): narrow Meta model fields via isinstance for type gate --- litellm/integrations/datadog/datadog_llm_obs.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index 4dcb40c7f8f..8b8fbc5d812 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -660,11 +660,14 @@ class DataDogLLMObsLogger(CustomBatchLogger): # Datadog prices llm/embedding spans from top-level meta.model_name / # model_provider. Those values already live in metadata; lift them so # embedding spans are not flagged "Partial cost — Unsupported model provider". - # Use TypedDict required-key access (not str(.get(...))) so basedpyright - # reportUnknownArgumentType / reportUnknownMemberType ceilings stay flat. - model_name: Final[str] = standard_logging_payload["model"] or "unknown" - provider_raw: Final[str | None] = standard_logging_payload["custom_llm_provider"] - model_provider: Final[str] = provider_raw if provider_raw is not None else "unknown" + # Narrow via isinstance so reportUnknownVariableType stays flat vs + # indexing a payload that basedpyright still treats as partially unknown. + _meta_model: Final[object] = payload_metadata.get("model_name", "unknown") + _meta_provider: Final[object] = payload_metadata.get("model_provider", "unknown") + model_name: Final[str] = _meta_model if isinstance(_meta_model, str) else "unknown" + model_provider: Final[str] = ( + _meta_provider if isinstance(_meta_provider, str) else "unknown" + ) meta: Final[Meta] = { "kind": span_kind, "input": input_meta, From b69bc5056730db09b2a8d59e8106b9f42b60c5f4 Mon Sep 17 00:00:00 2001 From: Quinn Xu Date: Thu, 15 Oct 2026 10:08:07 -0700 Subject: [PATCH 7/7] style(datadog): ruff-format Meta model field narrowing --- litellm/integrations/datadog/datadog_llm_obs.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index 8b8fbc5d812..a4782ff776c 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -665,9 +665,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): _meta_model: Final[object] = payload_metadata.get("model_name", "unknown") _meta_provider: Final[object] = payload_metadata.get("model_provider", "unknown") model_name: Final[str] = _meta_model if isinstance(_meta_model, str) else "unknown" - model_provider: Final[str] = ( - _meta_provider if isinstance(_meta_provider, str) else "unknown" - ) + model_provider: Final[str] = _meta_provider if isinstance(_meta_provider, str) else "unknown" meta: Final[Meta] = { "kind": span_kind, "input": input_meta,