From 533b93f7144cc968a66487e31fd5d5f1e228ae93 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 4 Dec 2023 14:49:59 -0800 Subject: [PATCH] (test) proxy: reading configs --- .../example_config_yaml/aliases_config.yaml | 30 +++++++++++++++++++ .../example_config_yaml/azure_config.yaml | 15 ++++++++++ .../example_config_yaml/langfuse_config.yaml | 7 +++++ .../example_config_yaml/load_balancer.yaml | 28 +++++++++++++++++ .../opentelemetry_config.yaml | 7 +++++ .../example_config_yaml/simple_config.yaml | 4 +++ litellm/tests/test_proxy_server.py | 10 +++++-- 7 files changed, 98 insertions(+), 3 deletions(-) create mode 100644 litellm/tests/example_config_yaml/aliases_config.yaml create mode 100644 litellm/tests/example_config_yaml/azure_config.yaml create mode 100644 litellm/tests/example_config_yaml/langfuse_config.yaml create mode 100644 litellm/tests/example_config_yaml/load_balancer.yaml create mode 100644 litellm/tests/example_config_yaml/opentelemetry_config.yaml create mode 100644 litellm/tests/example_config_yaml/simple_config.yaml diff --git a/litellm/tests/example_config_yaml/aliases_config.yaml b/litellm/tests/example_config_yaml/aliases_config.yaml new file mode 100644 index 00000000000..266f6cf22ee --- /dev/null +++ b/litellm/tests/example_config_yaml/aliases_config.yaml @@ -0,0 +1,30 @@ +model_list: + - model_name: text-davinci-003 + litellm_params: + model: ollama/zephyr + - model_name: gpt-4 + litellm_params: + model: ollama/llama2 + - model_name: gpt-3.5-turbo + litellm_params: + model: ollama/llama2 + temperature: 0.1 + max_tokens: 20 + + +# request to gpt-4, response from ollama/llama2 +# curl --location 'http://0.0.0.0:8000/chat/completions' \ +# --header 'Content-Type: application/json' \ +# --data ' { +# "model": "gpt-4", +# "messages": [ +# { +# "role": "user", +# "content": "what llm are you" +# } +# ], +# } +# ' +# + +# {"id":"chatcmpl-27c85cf0-ab09-4bcf-8cb1-0ee950520743","choices":[{"finish_reason":"stop","index":0,"message":{"content":" Hello! I'm just an AI, I don't have personal experiences or emotions like humans do. However, I can help you with any questions or tasks you may have! Is there something specific you'd like to know or discuss?","role":"assistant","_logprobs":null}}],"created":1700094955.373751,"model":"ollama/llama2","object":"chat.completion","system_fingerprint":null,"usage":{"prompt_tokens":12,"completion_tokens":47,"total_tokens":59},"_response_ms":8028.017999999999}% \ No newline at end of file diff --git a/litellm/tests/example_config_yaml/azure_config.yaml b/litellm/tests/example_config_yaml/azure_config.yaml new file mode 100644 index 00000000000..fd5865cd7c9 --- /dev/null +++ b/litellm/tests/example_config_yaml/azure_config.yaml @@ -0,0 +1,15 @@ +model_list: + - model_name: gpt-4-team1 + litellm_params: + model: azure/chatgpt-v-2 + api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ + api_version: "2023-05-15" + api_key: os.environ/AZURE_API_KEY + tpm: 20_000 + - model_name: gpt-4-team2 + litellm_params: + model: azure/gpt-4 + api_key: os.environ/AZURE_API_KEY + api_base: https://openai-gpt-4-test-v-2.openai.azure.com/ + tpm: 100_000 + diff --git a/litellm/tests/example_config_yaml/langfuse_config.yaml b/litellm/tests/example_config_yaml/langfuse_config.yaml new file mode 100644 index 00000000000..c2a77b5adc4 --- /dev/null +++ b/litellm/tests/example_config_yaml/langfuse_config.yaml @@ -0,0 +1,7 @@ +model_list: + - model_name: gpt-3.5-turbo + +litellm_settings: + drop_params: True + success_callback: ["langfuse"] # https://docs.litellm.ai/docs/observability/langfuse_integration + diff --git a/litellm/tests/example_config_yaml/load_balancer.yaml b/litellm/tests/example_config_yaml/load_balancer.yaml new file mode 100644 index 00000000000..502b90ff92a --- /dev/null +++ b/litellm/tests/example_config_yaml/load_balancer.yaml @@ -0,0 +1,28 @@ +litellm_settings: + drop_params: True + +# Model-specific settings +model_list: # use the same model_name for using the litellm router. LiteLLM will use the router between gpt-3.5-turbo + - model_name: gpt-3.5-turbo # litellm will + litellm_params: + model: gpt-3.5-turbo + api_key: sk-uj6F + tpm: 20000 # [OPTIONAL] REPLACE with your openai tpm + rpm: 3 # [OPTIONAL] REPLACE with your openai rpm + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo + api_key: sk-Imn + tpm: 20000 # [OPTIONAL] REPLACE with your openai tpm + rpm: 3 # [OPTIONAL] REPLACE with your openai rpm + - model_name: gpt-3.5-turbo + litellm_params: + model: openrouter/gpt-3.5-turbo + - model_name: mistral-7b-instruct + litellm_params: + model: mistralai/mistral-7b-instruct + +environment_variables: + REDIS_HOST: localhost + REDIS_PASSWORD: + REDIS_PORT: \ No newline at end of file diff --git a/litellm/tests/example_config_yaml/opentelemetry_config.yaml b/litellm/tests/example_config_yaml/opentelemetry_config.yaml new file mode 100644 index 00000000000..92d3454d7d5 --- /dev/null +++ b/litellm/tests/example_config_yaml/opentelemetry_config.yaml @@ -0,0 +1,7 @@ +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo + +general_settings: + otel: True # OpenTelemetry Logger this logs OTEL data to your collector diff --git a/litellm/tests/example_config_yaml/simple_config.yaml b/litellm/tests/example_config_yaml/simple_config.yaml new file mode 100644 index 00000000000..14b39a12518 --- /dev/null +++ b/litellm/tests/example_config_yaml/simple_config.yaml @@ -0,0 +1,4 @@ +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo \ No newline at end of file diff --git a/litellm/tests/test_proxy_server.py b/litellm/tests/test_proxy_server.py index 189cf60839c..4a10ecbf823 100644 --- a/litellm/tests/test_proxy_server.py +++ b/litellm/tests/test_proxy_server.py @@ -173,16 +173,20 @@ def test_load_router_config(): try: print("testing reading config") # this is a basic config.yaml with only a model - result = load_router_config(router=None, config_file_path="../proxy/example_config_yaml/simple_config.yaml") + result = load_router_config(router=None, config_file_path="example_config_yaml/simple_config.yaml") print(result) assert len(result[1]) == 1 # this is a load balancing config yaml - result = load_router_config(router=None, config_file_path="../proxy/example_config_yaml/azure_config.yaml") + result = load_router_config(router=None, config_file_path="example_config_yaml/azure_config.yaml") print(result) assert len(result[1]) == 2 + # config with general settings - custom callbacks + result = load_router_config(router=None, config_file_path="example_config_yaml/azure_config.yaml") + print(result) + assert len(result[1]) == 2 except Exception as e: pytest.fail("Proxy: Got exception reading config", e) -test_load_router_config() +# test_load_router_config()