From 62411d287012f8bcd475cff2b6bfc77d312f3815 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 8 Jun 2024 11:35:17 -0700 Subject: [PATCH] feat - low raw request on OTEL --- litellm/integrations/opentelemetry.py | 152 ++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index c8b34c477db..46c0c1a86ed 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -22,6 +22,7 @@ LITELLM_TRACER_NAME = os.getenv("OTEL_TRACER_NAME", "litellm") LITELLM_RESOURCE = { "service.name": os.getenv("OTEL_SERVICE_NAME", "litellm"), } +RAW_REQUEST_SPAN_NAME = "RAW_GENAI_REQUEST" @dataclass @@ -200,6 +201,7 @@ class OpenTelemetry(CustomLogger): ) _parent_context, parent_otel_span = self._get_span_context(kwargs) + # Span 1: Requst sent to litellm SDK span = self.tracer.start_span( name=self._get_span_name(kwargs), start_time=self._to_ns(start_time), @@ -208,6 +210,18 @@ class OpenTelemetry(CustomLogger): span.set_status(Status(StatusCode.OK)) self.set_attributes(span, kwargs, response_obj) span.end(end_time=self._to_ns(end_time)) + + # Span 2: Raw Request / Response to LLM + raw_request_span = self.tracer.start_span( + name=RAW_REQUEST_SPAN_NAME, + start_time=self._to_ns(start_time), + context=_parent_context, + ) + + raw_request_span.set_status(Status(StatusCode.OK)) + self.set_raw_request_attributes(raw_request_span, kwargs, response_obj) + raw_request_span.end(end_time=self._to_ns(end_time)) + if parent_otel_span is not None: parent_otel_span.end(end_time=self._to_ns(datetime.now())) @@ -324,6 +338,144 @@ class OpenTelemetry(CustomLogger): usage.get("prompt_tokens"), ) + def set_anthropic_raw_request_attributes(self, span: Span, kwargs, response_obj): + from opentelemetry.semconv.ai import SpanAttributes + + optional_params = kwargs.get("optional_params", {}) + litellm_params = kwargs.get("litellm_params", {}) or {} + custom_llm_provider = "anthropic" + + _raw_response = kwargs.get("original_response") + _additional_args = kwargs.get("additional_args", {}) or {} + complete_input_dict = _additional_args.get("complete_input_dict") + ############################################# + ########## LLM Request Attributes ########### + ############################################# + + # OTEL Attributes for the RAW Request to https://docs.anthropic.com/en/api/messages + if complete_input_dict: + if complete_input_dict.get("model"): + span.set_attribute( + SpanAttributes.LLM_REQUEST_MODEL, complete_input_dict.get("model") + ) + + if complete_input_dict.get("messages"): + for idx, prompt in enumerate(complete_input_dict.get("messages")): + span.set_attribute( + f"{SpanAttributes.LLM_PROMPTS}.{idx}.role", + prompt.get("role"), + ) + span.set_attribute( + f"{SpanAttributes.LLM_PROMPTS}.{idx}.content", + prompt.get("content"), + ) + if complete_input_dict.get("max_tokens"): + span.set_attribute( + SpanAttributes.LLM_REQUEST_MAX_TOKENS, + complete_input_dict.get("max_tokens"), + ) + + if complete_input_dict.get("temperature"): + span.set_attribute( + SpanAttributes.LLM_REQUEST_TEMPERATURE, + complete_input_dict.get("temperature"), + ) + + if complete_input_dict.get("top_p"): + span.set_attribute( + SpanAttributes.LLM_REQUEST_TOP_P, complete_input_dict.get("top_p") + ) + + if complete_input_dict.get("stream"): + span.set_attribute( + SpanAttributes.LLM_IS_STREAMING, complete_input_dict.get("stream") + ) + + if complete_input_dict.get("tools"): + span.set_attribute( + SpanAttributes.LLM_REQUEST_FUNCTIONS, + complete_input_dict.get("tools"), + ) + + if complete_input_dict.get("user"): + span.set_attribute( + SpanAttributes.LLM_USER, complete_input_dict.get("user") + ) + + ############################################# + ########## LLM Response Attributes ########## + ############################################# + if _raw_response: + # cast sr -> dict + import json + + _raw_response = json.loads(_raw_response) + + # The unique identifier for the completion. + if _raw_response.get("id"): + span.set_attribute("gen_ai.response.id", _raw_response.get("id")) + + # The model used to generate the response. + if _raw_response.get("model"): + span.set_attribute( + SpanAttributes.LLM_RESPONSE_MODEL, _raw_response.get("model") + ) + + if _raw_response.get("content"): + for idx, choice in enumerate(_raw_response.get("content")): + if choice.get("type"): + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.type", + choice.get("type"), + ) + + if choice.get("text"): + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.content", + choice.get("text"), + ) + + if choice.get("id"): + # https://docs.anthropic.com/en/docs/tool-use + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.id", + choice.get("id"), + ) + + if choice.get("name"): + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.name", + choice.get("name"), + ) + + if choice.get("input"): + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.input", + choice.get("input"), + ) + + pass + + def set_openai_raw_request_attributes(self, span: Span, kwargs, response_obj): + from opentelemetry.semconv.ai import SpanAttributes + + pass + + def set_default_raw_request_attributes(self, span: Span, kwargs, response_obj): + from opentelemetry.semconv.ai import SpanAttributes + + pass + + def set_raw_request_attributes(self, span: Span, kwargs, response_obj): + litellm_params = kwargs.get("litellm_params", {}) or {} + custom_llm_provider = litellm_params.get("custom_llm_provider", "Unknown") + if custom_llm_provider == "anthropic": + self.set_anthropic_raw_request_attributes(span, kwargs, response_obj) + elif custom_llm_provider == "openai": + self.set_openai_raw_request_attributes(span, kwargs, response_obj) + else: + self.set_default_raw_request_attributes(span, kwargs, response_obj) + def _to_ns(self, dt): return int(dt.timestamp() * 1e9)