From 25877493236a9a7a4b385d6b7f526bf5df64d11a Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Sat, 2 Dec 2023 14:24:38 -0800 Subject: [PATCH] (docs) proxy --- docs/my-website/docs/proxy/caching.md | 2 ++ docs/my-website/docs/proxy/cli.md | 1 + docs/my-website/docs/proxy/configs.md | 4 ++-- docs/my-website/docs/proxy/load_balancing.md | 4 +++- docs/my-website/docs/proxy/logging.md | 1 + docs/my-website/docs/proxy/virtual_keys.md | 1 + 6 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/my-website/docs/proxy/caching.md b/docs/my-website/docs/proxy/caching.md index f4b390029d0..d052102db9b 100644 --- a/docs/my-website/docs/proxy/caching.md +++ b/docs/my-website/docs/proxy/caching.md @@ -1,4 +1,6 @@ # Caching +Cache LLM Responses + Caching can be enabled by adding the `cache` key in the `config.yaml` #### Step 1: Add `cache` to the config.yaml ```yaml diff --git a/docs/my-website/docs/proxy/cli.md b/docs/my-website/docs/proxy/cli.md index 7aae584ff04..c9425b40252 100644 --- a/docs/my-website/docs/proxy/cli.md +++ b/docs/my-website/docs/proxy/cli.md @@ -1,4 +1,5 @@ # CLI Arguments +Cli arguments, --host, --port, --num_workers #### --host - **Default:** `'0.0.0.0'` diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index 77a1a65fe66..17c10b8425c 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -1,5 +1,5 @@ -# Config.yaml -The Config allows you to set the following params +# Proxy Config.yaml +Set model list, `api_base`, `api_key`, `temperature` & proxy server settings (`maaster-key`) | Param Name | Description | |----------------------|---------------------------------------------------------------| diff --git a/docs/my-website/docs/proxy/load_balancing.md b/docs/my-website/docs/proxy/load_balancing.md index 5c990c20413..d9215244edb 100644 --- a/docs/my-website/docs/proxy/load_balancing.md +++ b/docs/my-website/docs/proxy/load_balancing.md @@ -1,6 +1,8 @@ # Load Balancing - Multiple Instances of 1 model -Use this config to load balance between multiple instances of the same model. The proxy will handle routing requests (using LiteLLM's Router). **Set `rpm` in the config if you want maximize throughput** +Load balance multiple instances of the same model + +The proxy will handle routing requests (using LiteLLM's Router). **Set `rpm` in the config if you want maximize throughput** #### Example config requests with `model=gpt-3.5-turbo` will be routed across multiple instances of `azure/gpt-3.5-turbo` diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 8a9b67dfd88..0ad9480c189 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -1,4 +1,5 @@ # Logging - OpenTelemetry, Langfuse, ElasticSearch +Log Proxy Input, Output, Exceptions to Langfuse, OpenTelemetry ## Logging Proxy Input/Output - OpenTelemetry ### Step 1 Start OpenTelemetry Collecter Docker Container diff --git a/docs/my-website/docs/proxy/virtual_keys.md b/docs/my-website/docs/proxy/virtual_keys.md index 35501c5452b..275e206f9ac 100644 --- a/docs/my-website/docs/proxy/virtual_keys.md +++ b/docs/my-website/docs/proxy/virtual_keys.md @@ -1,5 +1,6 @@ # Cost Tracking & Virtual Keys +Track Spend and create virtual keys for the proxy Grant other's temporary access to your proxy, with keys that expire after a set duration.