From 4489c2b898d073ed6799cc4fd2f7b3143f87e0e2 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 15 Feb 2024 13:21:04 -0800 Subject: [PATCH 01/13] (v0) use custom loggers --- enterprise/callbacks/api_callback.py | 106 +++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 enterprise/callbacks/api_callback.py diff --git a/enterprise/callbacks/api_callback.py b/enterprise/callbacks/api_callback.py new file mode 100644 index 00000000000..4a1c27d41b2 --- /dev/null +++ b/enterprise/callbacks/api_callback.py @@ -0,0 +1,106 @@ +# callback to make a request to an API endpoint + +#### What this does #### +# On success, logs events to Promptlayer +import dotenv, os +import requests + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.caching import DualCache + +from typing import Literal, Union + +dotenv.load_dotenv() # Loading env variables using dotenv +import traceback + + +#### What this does #### +# On success + failure, log events to Supabase + +import dotenv, os +import requests + +dotenv.load_dotenv() # Loading env variables using dotenv +import traceback +import datetime, subprocess, sys +import litellm, uuid +from litellm._logging import print_verbose, verbose_logger + + +class GenericAPILogger: + # Class variables or attributes + def __init__(self, endpoint=None): + try: + verbose_logger.debug(f"in init GenericAPILogger, endpoint {endpoint}") + + pass + + except Exception as e: + print_verbose(f"Got exception on init GenericAPILogger client {str(e)}") + raise e + + # This is sync, because we run this in a separate thread. Running in a sepearate thread ensures it will never block an LLM API call + # Experience with s3, Langfuse shows that async logging events are complicated and can block LLM calls + def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose): + try: + verbose_logger.debug( + f"s3 Logging - Enters logging function for model {kwargs}" + ) + + # construct payload to send custom logger + # follows the same params as langfuse.py + litellm_params = kwargs.get("litellm_params", {}) + metadata = ( + litellm_params.get("metadata", {}) or {} + ) # if litellm_params['metadata'] == None + messages = kwargs.get("messages") + optional_params = kwargs.get("optional_params", {}) + call_type = kwargs.get("call_type", "litellm.completion") + cache_hit = kwargs.get("cache_hit", False) + usage = response_obj["usage"] + id = response_obj.get("id", str(uuid.uuid4())) + + # Build the initial payload + payload = { + "id": id, + "call_type": call_type, + "cache_hit": cache_hit, + "startTime": start_time, + "endTime": end_time, + "model": kwargs.get("model", ""), + "user": kwargs.get("user", ""), + "modelParameters": optional_params, + "messages": messages, + "response": response_obj, + "usage": usage, + "metadata": metadata, + } + + # Ensure everything in the payload is converted to str + for key, value in payload.items(): + try: + payload[key] = str(value) + except: + # non blocking if it can't cast to a str + pass + + import json + + payload = json.dumps(payload) + + print_verbose(f"\nGeneric Logger - Logging payload = {payload}") + + # make request to endpoint with payload + response = requests.post(self.endpoint, data=payload, headers=self.headers) + + response_status = response.status_code + response_text = response.text + + print_verbose( + f"Generic Logger - final response status = {response_status}, response text = {response_text}" + ) + return response + except Exception as e: + traceback.print_exc() + verbose_logger.debug(f"Generic - {str(e)}\n{traceback.format_exc()}") + pass From 4e8a94b916f80cf252a0a85db7c0b0ae2df61e74 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 15 Feb 2024 13:43:16 -0800 Subject: [PATCH 02/13] (feat) log with generic logger --- litellm/utils.py | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index c924aaf215f..37d5b750748 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -29,11 +29,12 @@ from dataclasses import ( dataclass, field, ) # for storing API inputs, outputs, and metadata -#import pkg_resources + +# import pkg_resources from importlib import resources # filename = pkg_resources.resource_filename(__name__, "llms/tokenizers") -filename = str(resources.files().joinpath("llms/tokenizers")) +filename = str(resources.files("llms").joinpath("tokenizers")) os.environ[ "TIKTOKEN_CACHE_DIR" ] = filename # use local copy of tiktoken b/c of - https://github.com/BerriAI/litellm/issues/1071 @@ -74,6 +75,10 @@ from .exceptions import ( BudgetExceededError, UnprocessableEntityError, ) + +# import enterprise features +from ..enterprise.callbacks.api_callback import GenericAPILogger + from typing import cast, List, Dict, Union, Optional, Literal, Any from .caching import Cache from concurrent.futures import ThreadPoolExecutor @@ -99,6 +104,7 @@ customLogger = None langFuseLogger = None dynamoLogger = None s3Logger = None +genericAPILogger = None llmonitorLogger = None aispendLogger = None berrispendLogger = None @@ -1361,6 +1367,35 @@ class Logging: user_id=kwargs.get("user", None), print_verbose=print_verbose, ) + if callback == "generic": + global genericAPILogger + verbose_logger.debug("reaches langfuse for success logging!") + kwargs = {} + for k, v in self.model_call_details.items(): + if ( + k != "original_response" + ): # copy.deepcopy raises errors as this could be a coroutine + kwargs[k] = v + # this only logs streaming once, complete_streaming_response exists i.e when stream ends + if self.stream: + verbose_logger.debug( + f"is complete_streaming_response in kwargs: {kwargs.get('complete_streaming_response', None)}" + ) + if complete_streaming_response is None: + break + else: + print_verbose("reaches langfuse for streaming logging!") + result = kwargs["complete_streaming_response"] + if genericAPILogger is None: + genericAPILogger = GenericAPILogger() + genericAPILogger.log_event( + kwargs=kwargs, + response_obj=result, + start_time=start_time, + end_time=end_time, + user_id=kwargs.get("user", None), + print_verbose=print_verbose, + ) if callback == "cache" and litellm.cache is not None: # this only logs streaming once, complete_streaming_response exists i.e when stream ends print_verbose("success_callback: reaches cache for logging!") From b3f54020178aa8ef288eb9814aec4ae81d7d71fb Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 15 Feb 2024 13:44:07 -0800 Subject: [PATCH 03/13] (feat) custom API callbacks --- enterprise/callbacks/api_callback.py | 15 +++++++++--- enterprise/callbacks/example_logging_api.py | 27 +++++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) create mode 100644 enterprise/callbacks/example_logging_api.py diff --git a/enterprise/callbacks/api_callback.py b/enterprise/callbacks/api_callback.py index 4a1c27d41b2..8c32ee86c1a 100644 --- a/enterprise/callbacks/api_callback.py +++ b/enterprise/callbacks/api_callback.py @@ -29,9 +29,11 @@ from litellm._logging import print_verbose, verbose_logger class GenericAPILogger: # Class variables or attributes - def __init__(self, endpoint=None): + def __init__(self, endpoint=None, headers=None): try: verbose_logger.debug(f"in init GenericAPILogger, endpoint {endpoint}") + self.endpoint = endpoint + self.headers = headers pass @@ -44,7 +46,7 @@ class GenericAPILogger: def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose): try: verbose_logger.debug( - f"s3 Logging - Enters logging function for model {kwargs}" + f"GenericAPILogger Logging - Enters logging function for model {kwargs}" ) # construct payload to send custom logger @@ -54,6 +56,7 @@ class GenericAPILogger: litellm_params.get("metadata", {}) or {} ) # if litellm_params['metadata'] == None messages = kwargs.get("messages") + cost = kwargs.get("response_cost", 0.0) optional_params = kwargs.get("optional_params", {}) call_type = kwargs.get("call_type", "litellm.completion") cache_hit = kwargs.get("cache_hit", False) @@ -74,6 +77,7 @@ class GenericAPILogger: "response": response_obj, "usage": usage, "metadata": metadata, + "cost": cost, } # Ensure everything in the payload is converted to str @@ -87,11 +91,14 @@ class GenericAPILogger: import json payload = json.dumps(payload) + data = { + "data": payload, + } - print_verbose(f"\nGeneric Logger - Logging payload = {payload}") + print_verbose(f"\nGeneric Logger - Logging payload = {data}") # make request to endpoint with payload - response = requests.post(self.endpoint, data=payload, headers=self.headers) + response = requests.post(self.endpoint, data=data, headers=self.headers) response_status = response.status_code response_text = response.text diff --git a/enterprise/callbacks/example_logging_api.py b/enterprise/callbacks/example_logging_api.py new file mode 100644 index 00000000000..f3c16299a07 --- /dev/null +++ b/enterprise/callbacks/example_logging_api.py @@ -0,0 +1,27 @@ +# this is an example endpoint to receive data from litellm +from fastapi import FastAPI, HTTPException, Request + +app = FastAPI() + + +@app.post("/log-event") +async def log_event(request: Request): + try: + # Assuming the incoming request has JSON data + data = await request.json() + print("Received request data:") + print(data) + + # Your additional logic can go here + # For now, just printing the received data + + return {"message": "Request received successfully"} + except Exception as e: + print(f"Error processing request: {str(e)}") + raise HTTPException(status_code=500, detail="Internal Server Error") + + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="127.0.0.1", port=8000) From 3e90acb7505b03c74c16a07231352b09ad11f2f4 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 15 Feb 2024 13:50:01 -0800 Subject: [PATCH 04/13] (feat) support headers for generic API logger --- enterprise/callbacks/api_callback.py | 16 +++++++++++++++- litellm/__init__.py | 1 + 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/enterprise/callbacks/api_callback.py b/enterprise/callbacks/api_callback.py index 8c32ee86c1a..8a8ea261779 100644 --- a/enterprise/callbacks/api_callback.py +++ b/enterprise/callbacks/api_callback.py @@ -31,10 +31,24 @@ class GenericAPILogger: # Class variables or attributes def __init__(self, endpoint=None, headers=None): try: - verbose_logger.debug(f"in init GenericAPILogger, endpoint {endpoint}") + if endpoint == None: + # check env for "GENERIC_LOGGER_ENDPOINT" + if os.getenv("GENERIC_LOGGER_ENDPOINT"): + # Do something with the endpoint + endpoint = os.getenv("GENERIC_LOGGER_ENDPOINT") + else: + # Handle the case when the endpoint is not found in the environment variables + raise ValueError( + f"endpoint not set for GenericAPILogger, GENERIC_LOGGER_ENDPOINT not found in environment variables" + ) + headers = headers or litellm.generic_logger_headers self.endpoint = endpoint self.headers = headers + verbose_logger.debug( + f"in init GenericAPILogger, endpoint {self.endpoint}, headers {self.headers}" + ) + pass except Exception as e: diff --git a/litellm/__init__.py b/litellm/__init__.py index 00d7488c94e..a7f232f7693 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -146,6 +146,7 @@ model_cost_map_url: str = "https://raw.githubusercontent.com/BerriAI/litellm/mai suppress_debug_info = False dynamodb_table_name: Optional[str] = None s3_callback_params: Optional[Dict] = None +generic_logger_headers: Optional[Dict] = None default_key_generate_params: Optional[Dict] = None upperbound_key_generate_params: Optional[Dict] = None default_team_settings: Optional[List] = None From 47b8715d25cfaac4cf4b9c0bfe37bea664dfd1be Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 15 Feb 2024 16:15:36 -0800 Subject: [PATCH 05/13] (feat) fix api callback imports --- ...pi_callback.py => generic_api_callback.py} | 4 +- litellm/tests/test_custom_api_logger.py | 45 +++++++++++++++++++ litellm/utils.py | 20 ++++++--- 3 files changed, 61 insertions(+), 8 deletions(-) rename enterprise/callbacks/{api_callback.py => generic_api_callback.py} (97%) create mode 100644 litellm/tests/test_custom_api_logger.py diff --git a/enterprise/callbacks/api_callback.py b/enterprise/callbacks/generic_api_callback.py similarity index 97% rename from enterprise/callbacks/api_callback.py rename to enterprise/callbacks/generic_api_callback.py index 8a8ea261779..309001e1bc2 100644 --- a/enterprise/callbacks/api_callback.py +++ b/enterprise/callbacks/generic_api_callback.py @@ -57,7 +57,9 @@ class GenericAPILogger: # This is sync, because we run this in a separate thread. Running in a sepearate thread ensures it will never block an LLM API call # Experience with s3, Langfuse shows that async logging events are complicated and can block LLM calls - def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose): + def log_event( + self, kwargs, response_obj, start_time, end_time, user_id, print_verbose + ): try: verbose_logger.debug( f"GenericAPILogger Logging - Enters logging function for model {kwargs}" diff --git a/litellm/tests/test_custom_api_logger.py b/litellm/tests/test_custom_api_logger.py new file mode 100644 index 00000000000..063dfa3e3e8 --- /dev/null +++ b/litellm/tests/test_custom_api_logger.py @@ -0,0 +1,45 @@ +import sys +import os +import io, asyncio + +# import logging +# logging.basicConfig(level=logging.DEBUG) +sys.path.insert(0, os.path.abspath("../..")) +print("Modified sys.path:", sys.path) + + +from litellm import completion +import litellm + +litellm.num_retries = 3 + +import time, random +import pytest + + +@pytest.mark.asyncio +async def test_custom_api_logging(): + try: + litellm.success_callback = ["generic"] + litellm.set_verbose = True + os.environ["GENERIC_LOGGER_ENDPOINT"] = "http://localhost:8000/log-event" + + print("Testing generic api logging") + + await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": f"This is a test"}], + max_tokens=10, + temperature=0.7, + user="ishaan-2", + ) + + except Exception as e: + pytest.fail(f"An exception occurred - {e}") + finally: + # post, close log file and verify + # Reset stdout to the original value + print("Passed! Testing async s3 logging") + + +# test_s3_logging() diff --git a/litellm/utils.py b/litellm/utils.py index 37d5b750748..d42e5f65570 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -12,6 +12,7 @@ import litellm import dotenv, json, traceback, threading, base64, ast import subprocess, os +from os.path import abspath, join, dirname import litellm, openai import itertools import random, uuid, requests @@ -33,11 +34,11 @@ from dataclasses import ( # import pkg_resources from importlib import resources -# filename = pkg_resources.resource_filename(__name__, "llms/tokenizers") -filename = str(resources.files("llms").joinpath("tokenizers")) -os.environ[ - "TIKTOKEN_CACHE_DIR" -] = filename # use local copy of tiktoken b/c of - https://github.com/BerriAI/litellm/issues/1071 +# # filename = pkg_resources.resource_filename(__name__, "llms/tokenizers") +# filename = str(resources.files().joinpath("llms/tokenizers")) +# os.environ[ +# "TIKTOKEN_CACHE_DIR" +# ] = filename # use local copy of tiktoken b/c of - https://github.com/BerriAI/litellm/issues/1071 encoding = tiktoken.get_encoding("cl100k_base") import importlib.metadata from ._logging import verbose_logger @@ -76,8 +77,13 @@ from .exceptions import ( UnprocessableEntityError, ) -# import enterprise features -from ..enterprise.callbacks.api_callback import GenericAPILogger +# Import Enterprise features +project_path = abspath(join(dirname(__file__), "..", "..")) +# Add the "enterprise" directory to sys.path +enterprise_path = abspath(join(project_path, "enterprise")) +sys.path.append(enterprise_path) +from enterprise.callbacks.generic_api_callback import GenericAPILogger + from typing import cast, List, Dict, Union, Optional, Literal, Any from .caching import Cache From aa333161a87187c7668b0ee1bdb8710e3b657cd0 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 15 Feb 2024 16:23:05 -0800 Subject: [PATCH 06/13] (feat) API custom callbacks --- enterprise/callbacks/example_logging_api.py | 4 ++++ enterprise/callbacks/generic_api_callback.py | 5 ++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/enterprise/callbacks/example_logging_api.py b/enterprise/callbacks/example_logging_api.py index f3c16299a07..57ea99a6748 100644 --- a/enterprise/callbacks/example_logging_api.py +++ b/enterprise/callbacks/example_logging_api.py @@ -7,6 +7,7 @@ app = FastAPI() @app.post("/log-event") async def log_event(request: Request): try: + print("Received /log-event request") # Assuming the incoming request has JSON data data = await request.json() print("Received request data:") @@ -18,6 +19,9 @@ async def log_event(request: Request): return {"message": "Request received successfully"} except Exception as e: print(f"Error processing request: {str(e)}") + import traceback + + traceback.print_exc() raise HTTPException(status_code=500, detail="Internal Server Error") diff --git a/enterprise/callbacks/generic_api_callback.py b/enterprise/callbacks/generic_api_callback.py index 309001e1bc2..076c13d5eef 100644 --- a/enterprise/callbacks/generic_api_callback.py +++ b/enterprise/callbacks/generic_api_callback.py @@ -106,15 +106,14 @@ class GenericAPILogger: import json - payload = json.dumps(payload) data = { "data": payload, } - + data = json.dumps(data) print_verbose(f"\nGeneric Logger - Logging payload = {data}") # make request to endpoint with payload - response = requests.post(self.endpoint, data=data, headers=self.headers) + response = requests.post(self.endpoint, json=data, headers=self.headers) response_status = response.status_code response_text = response.text From b6b5e0e7c29152a6be07dd4db78f58f8464d96f0 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 15 Feb 2024 17:06:16 -0800 Subject: [PATCH 07/13] (docs) use async callback APIs --- docs/my-website/docs/proxy/logging.md | 95 +++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 0c5fd6bb91d..03dbf520fd3 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -8,6 +8,7 @@ import TabItem from '@theme/TabItem'; Log Proxy Input, Output, Exceptions using Custom Callbacks, Langfuse, OpenTelemetry, LangFuse, DynamoDB, s3 Bucket - [Async Custom Callbacks](#custom-callback-class-async) +- [Async Custom Callback APIs](#custom-callback-apis-async) - [Logging to Langfuse](#logging-proxy-inputoutput---langfuse) - [Logging to s3 Buckets](#logging-proxy-inputoutput---s3-buckets) - [Logging to DynamoDB](#logging-proxy-inputoutput---dynamodb) @@ -297,6 +298,100 @@ ModelResponse( ``` +## Custom Callback APIs [Async] + +Use this if you: +- Want to use custom callbacks written in a non Python programming language +- Want your callbacks to run on a different microservice + +#### Step 1. Create your generic logging API endpoint +Set up a generic API endpoint that can receive data in JSON format. The data will be included within a "data" field. + +Your server should support the following Request format: + +```shell +curl --location https://your-domain.com/log-event \ + --request POST \ + --header "Content-Type: application/json" \ + --data '{ + "data": { + "id": "chatcmpl-8sgE89cEQ4q9biRtxMvDfQU1O82PT", + "call_type": "acompletion", + "cache_hit": "None", + "startTime": "2024-02-15 16:18:44.336280", + "endTime": "2024-02-15 16:18:45.045539", + "model": "gpt-3.5-turbo", + "user": "ishaan-2", + "modelParameters": "{'temperature': 0.7, 'max_tokens': 10, 'user': 'ishaan-2', 'extra_body': {}}", + "messages": "[{'role': 'user', 'content': 'This is a test'}]", + "response": "ModelResponse(id='chatcmpl-8sgE89cEQ4q9biRtxMvDfQU1O82PT', choices=[Choices(finish_reason='length', index=0, message=Message(content='Great! How can I assist you with this test', role='assistant'))], created=1708042724, model='gpt-3.5-turbo-0613', object='chat.completion', system_fingerprint=None, usage=Usage(completion_tokens=10, prompt_tokens=11, total_tokens=21))", + "usage": "Usage(completion_tokens=10, prompt_tokens=11, total_tokens=21)", + "metadata": "{}", + "cost": "3.65e-05" + } + }' +``` + +Reference FastAPI Python Server + +Here's a reference FastAPI Server that is compatible with LiteLLM Proxy: + +```python +# this is an example endpoint to receive data from litellm +from fastapi import FastAPI, HTTPException, Request + +app = FastAPI() + + +@app.post("/log-event") +async def log_event(request: Request): + try: + print("Received /log-event request") + # Assuming the incoming request has JSON data + data = await request.json() + print("Received request data:") + print(data) + + # Your additional logic can go here + # For now, just printing the received data + + return {"message": "Request received successfully"} + except Exception as e: + print(f"Error processing request: {str(e)}") + import traceback + + traceback.print_exc() + raise HTTPException(status_code=500, detail="Internal Server Error") + + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="127.0.0.1", port=8000) + + +``` + + +#### Step 2. Set your `GENERIC_LOGGER_ENDPOINT` to the endpoint + route we should send callback logs to + +```shell +os.environ["GENERIC_LOGGER_ENDPOINT"] = "http://localhost:8000/log-event" +``` + +#### Step 3. Create a `config.yaml` file and set `litellm_settings`: `success_callback` = ["generic"] + +Example litellm proxy config.yaml +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo +litellm_settings: + success_callback: ["generic"] +``` + +Start the LiteLLM Proxy and make a test request to verify the logs reached your callback API + ## Logging Proxy Input/Output - Langfuse We will use the `--config` to set `litellm.success_callback = ["langfuse"]` this will log all successfull LLM calls to langfuse From 92612c0d4e8cf5cb62fee8b07075afa159c02bf7 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 15 Feb 2024 17:16:36 -0800 Subject: [PATCH 08/13] (docs) custom callback apis --- docs/my-website/docs/proxy/logging.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 03dbf520fd3..3f5596fc9e6 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -300,6 +300,12 @@ ModelResponse( ## Custom Callback APIs [Async] +:::info + +This is an Enterprise only feature [Get Started with Enterprise here](https://github.com/BerriAI/litellm/tree/main/enterprise) + +::: + Use this if you: - Want to use custom callbacks written in a non Python programming language - Want your callbacks to run on a different microservice From 07afefea3472e2445cb680c61650f9e95ae353d4 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 15 Feb 2024 17:23:07 -0800 Subject: [PATCH 09/13] (chore) debug sys path docker error --- litellm/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/litellm/utils.py b/litellm/utils.py index d42e5f65570..63e5e3595e8 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -80,8 +80,11 @@ from .exceptions import ( # Import Enterprise features project_path = abspath(join(dirname(__file__), "..", "..")) # Add the "enterprise" directory to sys.path +verbose_logger.debug(f"current project_path: {project_path}") enterprise_path = abspath(join(project_path, "enterprise")) sys.path.append(enterprise_path) + +verbose_logger.debug(f"sys.path: {sys.path}") from enterprise.callbacks.generic_api_callback import GenericAPILogger From 56fba95b4a0c0d91a7067d2e7398a35dbb8b018e Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 15 Feb 2024 17:24:27 -0800 Subject: [PATCH 10/13] (fix) importing enterprise features --- litellm/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 63e5e3595e8..c05e2524aa2 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -85,8 +85,10 @@ enterprise_path = abspath(join(project_path, "enterprise")) sys.path.append(enterprise_path) verbose_logger.debug(f"sys.path: {sys.path}") -from enterprise.callbacks.generic_api_callback import GenericAPILogger - +try: + from enterprise.callbacks.generic_api_callback import GenericAPILogger +except Exception as e: + verbose_logger.debug(f"Exception import enterprise features {str(e)}") from typing import cast, List, Dict, Union, Optional, Literal, Any from .caching import Cache From cb7f380829539d079db8251bc95461f9abb40da2 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 15 Feb 2024 17:25:30 -0800 Subject: [PATCH 11/13] (fix) custom api testing --- litellm/tests/test_custom_api_logger.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/tests/test_custom_api_logger.py b/litellm/tests/test_custom_api_logger.py index 063dfa3e3e8..bddce9a0878 100644 --- a/litellm/tests/test_custom_api_logger.py +++ b/litellm/tests/test_custom_api_logger.py @@ -18,6 +18,7 @@ import pytest @pytest.mark.asyncio +@pytest.mark.skip(reason="new beta feature, will be testing in our ci/cd soon") async def test_custom_api_logging(): try: litellm.success_callback = ["generic"] From ec16b536a1f7590cface53a63995d38346a96aa8 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 15 Feb 2024 18:25:19 -0800 Subject: [PATCH 12/13] (fix) merge conflict --- litellm/utils.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index c05e2524aa2..6c286c87de5 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -34,11 +34,14 @@ from dataclasses import ( # import pkg_resources from importlib import resources -# # filename = pkg_resources.resource_filename(__name__, "llms/tokenizers") -# filename = str(resources.files().joinpath("llms/tokenizers")) -# os.environ[ -# "TIKTOKEN_CACHE_DIR" -# ] = filename # use local copy of tiktoken b/c of - https://github.com/BerriAI/litellm/issues/1071 +try: + filename = str( + resources.files().joinpath("llms/tokenizers") # type: ignore + ) # for python 3.8 and 3.12 +except: + filename = str( + resources.files(litellm).joinpath("llms/tokenizers") # for python 3.10 + ) # for python 3.10+ encoding = tiktoken.get_encoding("cl100k_base") import importlib.metadata from ._logging import verbose_logger From daa61cfdb6aaf7e497c4fafeb0cf42736f1d2be5 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 15 Feb 2024 18:34:53 -0800 Subject: [PATCH 13/13] (fix) merge conflicts --- litellm/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/utils.py b/litellm/utils.py index f195adfb002..d3efcea738f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -34,6 +34,7 @@ from dataclasses import ( # import pkg_resources from importlib import resources +# filename = pkg_resources.resource_filename(__name__, "llms/tokenizers") try: filename = str(