mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-11 03:38:38 +00:00
fix get_langfuse_logger_for_request
This commit is contained in:
Ishaan Jaff
parent
eca09ad89f
commit
64c3d27a23
3 changed files with 66 additions and 113 deletions
|
|
@ -4,18 +4,24 @@ import copy
|
|||
import inspect
|
||||
import os
|
||||
import traceback
|
||||
from typing import Any, Dict, Optional
|
||||
from typing import TYPE_CHECKING, Any, Dict, Optional
|
||||
|
||||
from packaging.version import Version
|
||||
from pydantic import BaseModel
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.litellm_logging import DynamicLoggingCache
|
||||
from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
|
||||
from litellm.secret_managers.main import str_to_bool
|
||||
from litellm.types.integrations.langfuse import *
|
||||
from litellm.types.utils import StandardCallbackDynamicParams, StandardLoggingPayload
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import DynamicLoggingCache
|
||||
else:
|
||||
DynamicLoggingCache = Any
|
||||
|
||||
|
||||
class LangFuseLogger:
|
||||
# Class variables or attributes
|
||||
|
|
@ -800,15 +806,51 @@ def log_requester_metadata(clean_metadata: dict):
|
|||
return returned_metadata
|
||||
|
||||
|
||||
def get_langfuse_logger_for_request(
|
||||
standard_callback_dynamic_params: StandardCallbackDynamicParams,
|
||||
in_memory_dynamic_logger_cache: DynamicLoggingCache,
|
||||
globalLangfuseLogger: Optional[LangFuseLogger] = None,
|
||||
) -> Optional[LangFuseLogger]:
|
||||
temp_langfuse_logger: Optional[LangFuseLogger] = globalLangfuseLogger
|
||||
|
||||
# get langfuse logging config to use for this request, based on standard_callback_dynamic_params
|
||||
_credentials = get_langfuse_logging_config(
|
||||
globalLangfuseLogger=globalLangfuseLogger,
|
||||
standard_callback_dynamic_params=standard_callback_dynamic_params,
|
||||
)
|
||||
credentials_dict = dict(_credentials)
|
||||
|
||||
# check if langfuse logger is already cached
|
||||
temp_langfuse_logger = in_memory_dynamic_logger_cache.get_cache(
|
||||
credentials=credentials_dict, service_name="langfuse"
|
||||
)
|
||||
|
||||
# if not cached, create a new langfuse logger and cache it
|
||||
if temp_langfuse_logger is None:
|
||||
temp_langfuse_logger = LangFuseLogger(
|
||||
langfuse_public_key=_credentials["langfuse_public_key"],
|
||||
langfuse_secret=_credentials["langfuse_secret"],
|
||||
langfuse_host=_credentials["langfuse_host"],
|
||||
)
|
||||
in_memory_dynamic_logger_cache.set_cache(
|
||||
credentials=credentials_dict,
|
||||
service_name="langfuse",
|
||||
logging_obj=temp_langfuse_logger,
|
||||
)
|
||||
|
||||
return temp_langfuse_logger
|
||||
|
||||
|
||||
def get_langfuse_logging_config(
|
||||
standard_callback_dynamic_params: StandardCallbackDynamicParams,
|
||||
globalLangfuseLogger: Optional[LangFuseLogger] = None,
|
||||
) -> LangfuseLoggingConfig:
|
||||
"""
|
||||
This function is used to get the Langfuse logging config for the Langfuse Logger.
|
||||
This function is used to get the Langfuse logging config to use for a given request.
|
||||
|
||||
It checks if the dynamic parameters are provided in the standard_callback_dynamic_params and uses them to get the Langfuse logging config.
|
||||
|
||||
If no dynamic parameters are provided, it uses the default values.
|
||||
If no dynamic parameters are provided, it uses the `globalLangfuseLogger` values
|
||||
"""
|
||||
# only use dynamic params if langfuse credentials are passed dynamically
|
||||
if _dynamic_langfuse_credentials_are_passed(standard_callback_dynamic_params):
|
||||
|
|
|
|||
|
|
@ -71,7 +71,7 @@ from ..integrations.gcs_bucket.gcs_bucket import GCSBucketLogger
|
|||
from ..integrations.greenscale import GreenscaleLogger
|
||||
from ..integrations.helicone import HeliconeLogger
|
||||
from ..integrations.lago import LagoLogger
|
||||
from ..integrations.langfuse import LangFuseLogger
|
||||
from ..integrations.langfuse import LangFuseLogger, get_langfuse_logger_for_request
|
||||
from ..integrations.langsmith import LangsmithLogger
|
||||
from ..integrations.litedebugger import LiteDebugger
|
||||
from ..integrations.literal_ai import LiteralAILogger
|
||||
|
|
@ -1136,74 +1136,13 @@ class Logging:
|
|||
print_verbose("reaches langfuse for streaming logging!")
|
||||
result = kwargs["complete_streaming_response"]
|
||||
|
||||
temp_langfuse_logger = langFuseLogger
|
||||
if langFuseLogger is None or (
|
||||
(
|
||||
self.standard_callback_dynamic_params.get(
|
||||
"langfuse_public_key"
|
||||
)
|
||||
is not None
|
||||
and self.standard_callback_dynamic_params.get(
|
||||
"langfuse_public_key"
|
||||
)
|
||||
!= langFuseLogger.public_key
|
||||
)
|
||||
or (
|
||||
self.standard_callback_dynamic_params.get(
|
||||
"langfuse_secret"
|
||||
)
|
||||
is not None
|
||||
and self.standard_callback_dynamic_params.get(
|
||||
"langfuse_secret"
|
||||
)
|
||||
!= langFuseLogger.secret_key
|
||||
)
|
||||
or (
|
||||
self.standard_callback_dynamic_params.get(
|
||||
"langfuse_host"
|
||||
)
|
||||
is not None
|
||||
and self.standard_callback_dynamic_params.get(
|
||||
"langfuse_host"
|
||||
)
|
||||
!= langFuseLogger.langfuse_host
|
||||
)
|
||||
):
|
||||
credentials = {
|
||||
"langfuse_public_key": self.standard_callback_dynamic_params.get(
|
||||
"langfuse_public_key"
|
||||
),
|
||||
"langfuse_secret": self.standard_callback_dynamic_params.get(
|
||||
"langfuse_secret"
|
||||
),
|
||||
"langfuse_host": self.standard_callback_dynamic_params.get(
|
||||
"langfuse_host"
|
||||
),
|
||||
}
|
||||
temp_langfuse_logger = (
|
||||
in_memory_dynamic_logger_cache.get_cache(
|
||||
credentials=credentials, service_name="langfuse"
|
||||
)
|
||||
)
|
||||
if temp_langfuse_logger is None:
|
||||
temp_langfuse_logger = LangFuseLogger(
|
||||
langfuse_public_key=self.standard_callback_dynamic_params.get(
|
||||
"langfuse_public_key"
|
||||
),
|
||||
langfuse_secret=self.standard_callback_dynamic_params.get(
|
||||
"langfuse_secret"
|
||||
),
|
||||
langfuse_host=self.standard_callback_dynamic_params.get(
|
||||
"langfuse_host"
|
||||
),
|
||||
)
|
||||
in_memory_dynamic_logger_cache.set_cache(
|
||||
credentials=credentials,
|
||||
service_name="langfuse",
|
||||
logging_obj=temp_langfuse_logger,
|
||||
)
|
||||
if temp_langfuse_logger is not None:
|
||||
_response = temp_langfuse_logger.log_event(
|
||||
langfuse_logger_to_use = get_langfuse_logger_for_request(
|
||||
globalLangfuseLogger=langFuseLogger,
|
||||
standard_callback_dynamic_params=self.standard_callback_dynamic_params,
|
||||
in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache,
|
||||
)
|
||||
if langfuse_logger_to_use is not None:
|
||||
_response = langfuse_logger_to_use.log_event(
|
||||
kwargs=kwargs,
|
||||
response_obj=result,
|
||||
start_time=start_time,
|
||||
|
|
|
|||
|
|
@ -1,48 +1,20 @@
|
|||
model_list:
|
||||
################################################################################
|
||||
# Azure
|
||||
- model_name: gpt-4o-mini
|
||||
litellm_params:
|
||||
model: azure/gpt-4o-mini
|
||||
api_base: https://amazin-prod.openai.azure.com
|
||||
api_key: "os.environ/AZURE_GPT_4O"
|
||||
deployment_id: gpt-4o-mini
|
||||
- model_name: gpt-4o
|
||||
litellm_params:
|
||||
model: azure/gpt-4o
|
||||
api_base: https://very-cool-prod.openai.azure.com
|
||||
api_key: "os.environ/AZURE_GPT_4O"
|
||||
deployment_id: gpt-4o
|
||||
model: gpt-4o
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
tpm: 1000000
|
||||
rpm: 10000
|
||||
|
||||
|
||||
################################################################################
|
||||
# Fireworks
|
||||
- model_name: fireworks-llama-v3p1-405b-instruct
|
||||
litellm_params:
|
||||
model: fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct
|
||||
api_key: "os.environ/FIREWORKS"
|
||||
- model_name: fireworks-llama-v3p1-70b-instruct
|
||||
litellm_params:
|
||||
model: fireworks_ai/accounts/fireworks/models/llama-v3p1-70b-instruct
|
||||
api_key: "os.environ/FIREWORKS"
|
||||
|
||||
general_settings:
|
||||
alerting_threshold: 300 # sends alerts if requests hang for 5min+ and responses take 5min+
|
||||
litellm_settings: # module level litellm settings - https://github.com/BerriAI/litellm/blob/main/litellm/__init__.py
|
||||
success_callback: ["prometheus"]
|
||||
service_callback: ["prometheus_system"]
|
||||
drop_params: False # Raise an exception if the openai param being passed in isn't supported.
|
||||
cache: false
|
||||
default_internal_user_params:
|
||||
user_role: os.environ/DEFAULT_USER_ROLE
|
||||
# master key is set via env var
|
||||
# master_key: #######
|
||||
proxy_batch_write_at: 60 # Batch write spend updates every 60s
|
||||
|
||||
success_callback: ["s3"]
|
||||
s3_callback_params:
|
||||
s3_bucket_name: logs-bucket-litellm # AWS Bucket Name for S3
|
||||
s3_region_name: us-west-2 # AWS Region Name for S3
|
||||
s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # us os.environ/<variable name> to pass environment variables. This is AWS Access Key ID for S3
|
||||
s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3
|
||||
s3_path: my-test-path # [OPTIONAL] set path in bucket you want to write logs to
|
||||
s3_endpoint_url: https://s3.amazonaws.com # [OPTIONAL] S3 endpoint URL, if you want to use Backblaze/cloudflare s3 buckets
|
||||
litellm_settings:
|
||||
success_callback: ["langfuse"]
|
||||
|
||||
router_settings:
|
||||
routing_strategy: simple-shuffle # "simple-shuffle" shown to result in highest throughput. https://docs.litellm.ai/docs/proxy/configs#load-balancing
|
||||
# https://docs.litellm.ai/docs/proxy/reliability#default-fallbacks
|
||||
default_fallbacks: ["gpt-4o-2024-08-06", "claude-3-5-sonnet-20240620"]
|
||||
fallbacks: [{"gpt-4o-2024-08-06": ["claude-3-5-sonnet-20240620"]}, {"gpt-4o-2024-05-13": ["claude-3-5-sonnet-20240620"]}]
|
||||
Loading…
Add table
Reference in a new issue