diff --git a/docs/my-website/docs/debugging/local_debugging.md b/docs/my-website/docs/debugging/local_debugging.md index cd2b4e4449e..73cb0d27daf 100644 --- a/docs/my-website/docs/debugging/local_debugging.md +++ b/docs/my-website/docs/debugging/local_debugging.md @@ -1,5 +1,5 @@ # Local Debugging -There's 2 ways to do local debugging - `litellm.set_verbose=True` and by passing in a custom function `completion(...logger_fn=)` +There's 2 ways to do local debugging - `litellm.set_verbose=True` and by passing in a custom function `completion(...logger_fn=)`. Warning: Make sure to not use `set_verbose` in production. It logs API keys, which might end up in log files. ## Set Verbose @@ -60,4 +60,4 @@ response = completion("command-nightly", messages, logger_fn=my_custom_logging_f Text us @ +17708783106 or Join the [Discord](https://discord.com/invite/wuPM9dRgDw). -We promise to help you in `lite`ning speed ❤️ \ No newline at end of file +We promise to help you in `lite`ning speed ❤️ diff --git a/docs/my-website/docs/proxy_server.md b/docs/my-website/docs/proxy_server.md index d2a9d521ff9..5f80789a1e8 100644 --- a/docs/my-website/docs/proxy_server.md +++ b/docs/my-website/docs/proxy_server.md @@ -208,6 +208,86 @@ user_proxy.initiate_chat(assistant, message="Plot a chart of META and TESLA stoc Credits [@victordibia](https://github.com/microsoft/autogen/issues/45#issuecomment-1749921972) for this tutorial. + + + +```python +from autogen import AssistantAgent, GroupChatManager, UserProxyAgent +from autogen.agentchat import GroupChat +config_list = [ + { + "model": "ollama/mistralorca", + "api_base": "http://localhost:8000", # litellm compatible endpoint + "api_type": "open_ai", + "api_key": "NULL", # just a placeholder + } +] +llm_config = {"config_list": config_list, "seed": 42} + +code_config_list = [ + { + "model": "ollama/phind-code", + "api_base": "http://localhost:8000", # litellm compatible endpoint + "api_type": "open_ai", + "api_key": "NULL", # just a placeholder + } +] + +code_config = {"config_list": code_config_list, "seed": 42} + +admin = UserProxyAgent( + name="Admin", + system_message="A human admin. Interact with the planner to discuss the plan. Plan execution needs to be approved by this admin.", + llm_config=llm_config, + code_execution_config=False, +) + + +engineer = AssistantAgent( + name="Engineer", + llm_config=code_config, + system_message="""Engineer. You follow an approved plan. You write python/shell code to solve tasks. Wrap the code in a code block that specifies the script type. The user can't modify your code. So do not suggest incomplete code which requires others to modify. Don't use a code block if it's not intended to be executed by the executor. +Don't include multiple code blocks in one response. Do not ask others to copy and paste the result. Check the execution result returned by the executor. +If the result indicates there is an error, fix the error and output the code again. Suggest the full code instead of partial code or code changes. If the error can't be fixed or if the task is not solved even after the code is executed successfully, analyze the problem, revisit your assumption, collect additional info you need, and think of a different approach to try. +""", +) +planner = AssistantAgent( + name="Planner", + system_message="""Planner. Suggest a plan. Revise the plan based on feedback from admin and critic, until admin approval. +The plan may involve an engineer who can write code and a scientist who doesn't write code. +Explain the plan first. Be clear which step is performed by an engineer, and which step is performed by a scientist. +""", + llm_config=llm_config, +) +executor = UserProxyAgent( + name="Executor", + system_message="Executor. Execute the code written by the engineer and report the result.", + human_input_mode="NEVER", + llm_config=llm_config, + code_execution_config={"last_n_messages": 3, "work_dir": "paper"}, +) +critic = AssistantAgent( + name="Critic", + system_message="Critic. Double check plan, claims, code from other agents and provide feedback. Check whether the plan includes adding verifiable info such as source URL.", + llm_config=llm_config, +) +groupchat = GroupChat( + agents=[admin, engineer, planner, executor, critic], + messages=[], + max_round=50, +) +manager = GroupChatManager(groupchat=groupchat, llm_config=llm_config) + + +admin.initiate_chat( + manager, + message=""" +""", +) +``` + +Credits [@Nathan](https://gist.github.com/CUexter) for this tutorial. + ```python diff --git a/litellm/llms/baseten.py b/litellm/llms/baseten.py index 8f24e129ea4..aecacd84ff1 100644 --- a/litellm/llms/baseten.py +++ b/litellm/llms/baseten.py @@ -121,7 +121,7 @@ def completion( sum_logprob = 0 for token in completion_response[0]["details"]["tokens"]: sum_logprob += token["logprob"] - model_response["choices"][0]["message"]["logprobs"] = sum_logprob + model_response["choices"][0]["message"]._logprobs = sum_logprob else: raise BasetenError( message=f"Unable to parse response. Original response: {response.text}", diff --git a/litellm/llms/huggingface_restapi.py b/litellm/llms/huggingface_restapi.py index c63c3115007..b3c3e5e38db 100644 --- a/litellm/llms/huggingface_restapi.py +++ b/litellm/llms/huggingface_restapi.py @@ -141,7 +141,6 @@ def completion( litellm_params=None, logger_fn=None, ): - print(f'headers inside hf rest api: {headers}') headers = validate_environment(api_key, headers) task = get_hf_task_for_model(model) print_verbose(f"{model}, {task}") @@ -254,8 +253,6 @@ def completion( ## Some servers might return streaming responses even though stream was not set to true. (e.g. Baseten) is_streamed = False - print(f"response keys: {response.__dict__.keys()}") - print(f"response keys: {response.__dict__['headers']}") if response.__dict__['headers']["Content-Type"] == "text/event-stream": is_streamed = True @@ -313,7 +310,7 @@ def completion( sum_logprob = 0 for token in completion_response[0]["details"]["tokens"]: sum_logprob += token["logprob"] - model_response["choices"][0]["message"]["logprobs"] = sum_logprob + model_response["choices"][0]["message"]._logprob = sum_logprob if "best_of" in optional_params and optional_params["best_of"] > 1: if "details" in completion_response[0] and "best_of_sequences" in completion_response[0]["details"]: choices_list = [] @@ -337,9 +334,14 @@ def completion( prompt_tokens = len( encoding.encode(input_text) ) ##[TODO] use the llama2 tokenizer here - completion_tokens = len( - encoding.encode(model_response["choices"][0]["message"].get("content", "")) - ) ##[TODO] use the llama2 tokenizer here + print_verbose(f'output: {model_response["choices"][0]["message"]}') + output_text = model_response["choices"][0]["message"].get("content", "") + if output_text is not None and len(output_text) > 0: + completion_tokens = len( + encoding.encode(model_response["choices"][0]["message"].get("content", "")) + ) ##[TODO] use the llama2 tokenizer here + else: + completion_tokens = 0 model_response["created"] = time.time() model_response["model"] = model diff --git a/litellm/main.py b/litellm/main.py index e24c5af73b2..6512ded0ade 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -729,7 +729,6 @@ def completion( headers or litellm.headers ) - print(f'headers before hf rest api: {hf_headers}') model_response = huggingface_restapi.completion( model=model, messages=messages, diff --git a/litellm/proxy/llm.py b/litellm/proxy/llm.py index 878131697eb..85d34d06cd2 100644 --- a/litellm/proxy/llm.py +++ b/litellm/proxy/llm.py @@ -9,6 +9,7 @@ import backoff import openai.error import litellm +from litellm.utils import trim_messages import litellm.exceptions cost_dict: Dict[str, Dict[str, float]] = defaultdict(dict) @@ -113,7 +114,7 @@ def litellm_completion(data: Dict, user_api_base: Optional[str], user_headers: Optional[dict], user_debug: bool) -> litellm.ModelResponse: - try: + try: global debug debug = user_debug if user_model: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 56375c158c4..5145548df93 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2,6 +2,7 @@ import sys, os, platform, time, copy import threading import shutil, random, traceback +messages = [] sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path - for litellm local dev @@ -72,6 +73,7 @@ print() import litellm from fastapi import FastAPI, Request from fastapi.routing import APIRouter +from fastapi.encoders import jsonable_encoder from fastapi.responses import StreamingResponse, FileResponse from fastapi.middleware.cors import CORSMiddleware import json @@ -111,6 +113,13 @@ def print_verbose(print_statement): print(print_statement) +def find_avatar_url(role): + role = role.replace(" ", "%20") + avatar_filename = f"avatars/{role}.png" + avatar_url = f"/static/{avatar_filename}" + return avatar_url + + def usage_telemetry( feature: str): # helps us know if people are using this feature. Set `litellm --telemetry False` to your cli call to turn this off if user_telemetry: @@ -461,24 +470,21 @@ def model_list(): ) -@router.post("/v1/completions") @router.post("/completions") async def completion(request: Request): data = await request.json() - print_verbose(f"data passed in: {data}") return litellm_completion(data=data, type="completion", user_model=user_model, user_temperature=user_temperature, user_max_tokens=user_max_tokens, user_api_base=user_api_base, user_headers=user_headers, user_debug=user_debug) -@router.post("/v1/chat/completions") @router.post("/chat/completions") async def chat_completion(request: Request): data = await request.json() print_verbose(f"data passed in: {data}") return litellm_completion(data, type="chat_completion", user_model=user_model, - user_temperature=user_temperature, user_max_tokens=user_max_tokens, - user_api_base=user_api_base, user_headers=user_headers, user_debug=user_debug) + user_temperature=user_temperature, user_max_tokens=user_max_tokens, + user_api_base=user_api_base, user_headers=user_headers, user_debug=user_debug) def print_cost_logs(): diff --git a/litellm/utils.py b/litellm/utils.py index 478eeda36ac..a077dbc7e45 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -119,7 +119,7 @@ class Message(OpenAIObject): super(Message, self).__init__(**params) self.content = content self.role = role - self.logprobs = logprobs + self._logprobs = logprobs class Delta(OpenAIObject): def __init__(self, content=None, logprobs=None, role=None, **params):