diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index d4bddd9a0ad..6279d0e2963 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -3,20 +3,14 @@ model_list: litellm_params: model: openai/fake api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railwaz.app/ + api_base: https://exampleopenaiendpoint-production.up.railway.app/ - model_name: fireworks-llama-v3-70b-instruct litellm_params: model: fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct api_key: "os.environ/FIREWORKS" - # provider specific wildcard routing - - model_name: "anthropic/*" + - model_name: "*" litellm_params: - model: "anthropic/*" - api_key: os.environ/ANTHROPIC_API_KEY - - model_name: "groq/*" - litellm_params: - model: "groq/*" - api_key: os.environ/GROQ_API_KEY + model: "*" - model_name: "*" litellm_params: model: openai/* @@ -25,37 +19,18 @@ model_list: litellm_params: model: mistral/mistral-small-latest api_key: "os.environ/MISTRAL_API_KEY" - - model_name: tts + - model_name: gemini-1.5-pro-001 litellm_params: - model: openai/tts-1 - api_key: "os.environ/OPENAI_API_KEY" - model_info: - mode: audio_speech - - -# for /files endpoints -files_settings: - - custom_llm_provider: azure - api_base: https://exampleopenaiendpoint-production.up.railway.app - api_key: fake-key - api_version: "2023-03-15-preview" - - custom_llm_provider: openai - api_key: os.environ/OPENAI_API_KEY + model: vertex_ai_beta/gemini-1.5-pro-001 + vertex_project: "adroit-crow-413218" + vertex_location: "us-central1" + vertex_credentials: "adroit-crow-413218-a956eef1a2a8.json" + # Add path to service account.json general_settings: master_key: sk-1234 - pass_through_endpoints: - - path: "/v1/rerank" # route you want to add to LiteLLM Proxy Server - target: "https://api.cohere.com/v1/rerank" # URL this route should forward requests to - headers: # headers to forward to this URL - content-type: application/json # (Optional) Extra Headers to pass to this endpoint - accept: application/json - forward_headers: True - litellm_settings: callbacks: ["otel"] # 👈 KEY CHANGE - success_callback: ["prometheus"] - failure_callback: ["prometheus"] \ No newline at end of file diff --git a/litellm/proxy/tests/test_gemini_context_caching.py b/litellm/proxy/tests/test_gemini_context_caching.py new file mode 100644 index 00000000000..40caaf1eafc --- /dev/null +++ b/litellm/proxy/tests/test_gemini_context_caching.py @@ -0,0 +1,38 @@ +import datetime + +import openai +import vertexai +from vertexai.generative_models import Content, Part +from vertexai.preview import caching +from vertexai.preview.generative_models import GenerativeModel + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") +vertexai.init(project="adroit-crow-413218", location="us-central1") +print("creating cached content") +contents_here: list[Content] = [ + Content(role="user", parts=[Part.from_text("huge string of text here" * 10000)]) +] +cached_content = caching.CachedContent.create( + model_name="gemini-1.5-pro-001", + contents=contents_here, + expire_time=datetime.datetime(2024, 8, 10), +) + +created_Caches = caching.CachedContent.list() + +print("created_Caches contents=", created_Caches) + +response = client.chat.completions.create( # type: ignore + model="gemini-1.5-pro-001", + max_tokens=8192, + messages=[ + { + "role": "user", + "content": "quote all everything above this message", + }, + ], + temperature="0.7", + extra_body={"cached_content": cached_content.resource_name}, +) + +print("response from proxy", response)