diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index fa52b3d7a2f..edaf3527c5d 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -4,18 +4,24 @@ import copy import inspect import os import traceback -from typing import Any, Dict, Optional +from typing import TYPE_CHECKING, Any, Dict, Optional from packaging.version import Version from pydantic import BaseModel import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.litellm_logging import DynamicLoggingCache from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info from litellm.secret_managers.main import str_to_bool from litellm.types.integrations.langfuse import * from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPayload +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import DynamicLoggingCache +else: + DynamicLoggingCache = Any + class LangFuseLogger: # Class variables or attributes @@ -800,15 +806,51 @@ def log_requester_metadata(clean_metadata: dict): return returned_metadata +def get_langfuse_logger_for_request( + standard_callback_dynamic_params: StandardCallbackDynamicParams, + in_memory_dynamic_logger_cache: DynamicLoggingCache, + globalLangfuseLogger: Optional[LangFuseLogger] = None, +) -> Optional[LangFuseLogger]: + temp_langfuse_logger: Optional[LangFuseLogger] = globalLangfuseLogger + + # get langfuse logging config to use for this request, based on standard_callback_dynamic_params + _credentials = get_langfuse_logging_config( + globalLangfuseLogger=globalLangfuseLogger, + standard_callback_dynamic_params=standard_callback_dynamic_params, + ) + credentials_dict = dict(_credentials) + + # check if langfuse logger is already cached + temp_langfuse_logger = in_memory_dynamic_logger_cache.get_cache( + credentials=credentials_dict, service_name="langfuse" + ) + + # if not cached, create a new langfuse logger and cache it + if temp_langfuse_logger is None: + temp_langfuse_logger = LangFuseLogger( + langfuse_public_key=_credentials["langfuse_public_key"], + langfuse_secret=_credentials["langfuse_secret"], + langfuse_host=_credentials["langfuse_host"], + ) + in_memory_dynamic_logger_cache.set_cache( + credentials=credentials_dict, + service_name="langfuse", + logging_obj=temp_langfuse_logger, + ) + + return temp_langfuse_logger + + def get_langfuse_logging_config( standard_callback_dynamic_params: StandardCallbackDynamicParams, globalLangfuseLogger: Optional[LangFuseLogger] = None, ) -> LangfuseLoggingConfig: """ - This function is used to get the Langfuse logging config for the Langfuse Logger. + This function is used to get the Langfuse logging config to use for a given request. + It checks if the dynamic parameters are provided in the standard_callback_dynamic_params and uses them to get the Langfuse logging config. - If no dynamic parameters are provided, it uses the default values. + If no dynamic parameters are provided, it uses the `globalLangfuseLogger` values """ # only use dynamic params if langfuse credentials are passed dynamically if _dynamic_langfuse_credentials_are_passed(standard_callback_dynamic_params): diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index d17b2c79084..6643a5ebdeb 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -71,7 +71,7 @@ from ..integrations.gcs_bucket.gcs_bucket import GCSBucketLogger from ..integrations.greenscale import GreenscaleLogger from ..integrations.helicone import HeliconeLogger from ..integrations.lago import LagoLogger -from ..integrations.langfuse import LangFuseLogger +from ..integrations.langfuse import LangFuseLogger, get_langfuse_logger_for_request from ..integrations.langsmith import LangsmithLogger from ..integrations.litedebugger import LiteDebugger from ..integrations.literal_ai import LiteralAILogger @@ -1136,74 +1136,13 @@ class Logging: print_verbose("reaches langfuse for streaming logging!") result = kwargs["complete_streaming_response"] - temp_langfuse_logger = langFuseLogger - if langFuseLogger is None or ( - ( - self.standard_callback_dynamic_params.get( - "langfuse_public_key" - ) - is not None - and self.standard_callback_dynamic_params.get( - "langfuse_public_key" - ) - != langFuseLogger.public_key - ) - or ( - self.standard_callback_dynamic_params.get( - "langfuse_secret" - ) - is not None - and self.standard_callback_dynamic_params.get( - "langfuse_secret" - ) - != langFuseLogger.secret_key - ) - or ( - self.standard_callback_dynamic_params.get( - "langfuse_host" - ) - is not None - and self.standard_callback_dynamic_params.get( - "langfuse_host" - ) - != langFuseLogger.langfuse_host - ) - ): - credentials = { - "langfuse_public_key": self.standard_callback_dynamic_params.get( - "langfuse_public_key" - ), - "langfuse_secret": self.standard_callback_dynamic_params.get( - "langfuse_secret" - ), - "langfuse_host": self.standard_callback_dynamic_params.get( - "langfuse_host" - ), - } - temp_langfuse_logger = ( - in_memory_dynamic_logger_cache.get_cache( - credentials=credentials, service_name="langfuse" - ) - ) - if temp_langfuse_logger is None: - temp_langfuse_logger = LangFuseLogger( - langfuse_public_key=self.standard_callback_dynamic_params.get( - "langfuse_public_key" - ), - langfuse_secret=self.standard_callback_dynamic_params.get( - "langfuse_secret" - ), - langfuse_host=self.standard_callback_dynamic_params.get( - "langfuse_host" - ), - ) - in_memory_dynamic_logger_cache.set_cache( - credentials=credentials, - service_name="langfuse", - logging_obj=temp_langfuse_logger, - ) - if temp_langfuse_logger is not None: - _response = temp_langfuse_logger.log_event( + langfuse_logger_to_use = get_langfuse_logger_for_request( + globalLangfuseLogger=langFuseLogger, + standard_callback_dynamic_params=self.standard_callback_dynamic_params, + in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache, + ) + if langfuse_logger_to_use is not None: + _response = langfuse_logger_to_use.log_event( kwargs=kwargs, response_obj=result, start_time=start_time, diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index bae738c735d..8f22d0d780c 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,48 +1,20 @@ model_list: - ################################################################################ - # Azure - - model_name: gpt-4o-mini - litellm_params: - model: azure/gpt-4o-mini - api_base: https://amazin-prod.openai.azure.com - api_key: "os.environ/AZURE_GPT_4O" - deployment_id: gpt-4o-mini - model_name: gpt-4o litellm_params: - model: azure/gpt-4o - api_base: https://very-cool-prod.openai.azure.com - api_key: "os.environ/AZURE_GPT_4O" - deployment_id: gpt-4o + model: gpt-4o + api_key: os.environ/OPENAI_API_KEY + tpm: 1000000 + rpm: 10000 + - ################################################################################ - # Fireworks - - model_name: fireworks-llama-v3p1-405b-instruct - litellm_params: - model: fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct - api_key: "os.environ/FIREWORKS" - - model_name: fireworks-llama-v3p1-70b-instruct - litellm_params: - model: fireworks_ai/accounts/fireworks/models/llama-v3p1-70b-instruct - api_key: "os.environ/FIREWORKS" - general_settings: - alerting_threshold: 300 # sends alerts if requests hang for 5min+ and responses take 5min+ -litellm_settings: # module level litellm settings - https://github.com/BerriAI/litellm/blob/main/litellm/__init__.py - success_callback: ["prometheus"] - service_callback: ["prometheus_system"] - drop_params: False # Raise an exception if the openai param being passed in isn't supported. - cache: false - default_internal_user_params: - user_role: os.environ/DEFAULT_USER_ROLE + # master key is set via env var + # master_key: ####### + proxy_batch_write_at: 60 # Batch write spend updates every 60s - success_callback: ["s3"] - s3_callback_params: - s3_bucket_name: logs-bucket-litellm # AWS Bucket Name for S3 - s3_region_name: us-west-2 # AWS Region Name for S3 - s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # us os.environ/ to pass environment variables. This is AWS Access Key ID for S3 - s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3 - s3_path: my-test-path # [OPTIONAL] set path in bucket you want to write logs to - s3_endpoint_url: https://s3.amazonaws.com # [OPTIONAL] S3 endpoint URL, if you want to use Backblaze/cloudflare s3 buckets +litellm_settings: + success_callback: ["langfuse"] -router_settings: - routing_strategy: simple-shuffle # "simple-shuffle" shown to result in highest throughput. https://docs.litellm.ai/docs/proxy/configs#load-balancing + # https://docs.litellm.ai/docs/proxy/reliability#default-fallbacks + default_fallbacks: ["gpt-4o-2024-08-06", "claude-3-5-sonnet-20240620"] + fallbacks: [{"gpt-4o-2024-08-06": ["claude-3-5-sonnet-20240620"]}, {"gpt-4o-2024-05-13": ["claude-3-5-sonnet-20240620"]}] \ No newline at end of file