From ef815f3a849097438a068173ac7ded56c81ee9db Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 8 Oct 2024 07:57:04 +0530 Subject: [PATCH 1/2] (docs) add remaining litellm settings on configs.md doc (#6108) * docs add litellm settings configs * docs langfuse tags on config --- docs/my-website/docs/proxy/configs.md | 80 +++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index e73db18ad2a..f4e43aa80e9 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -599,12 +599,64 @@ model_list: additionalProp1: {} litellm_settings: + # Logging/Callback settings success_callback: ["langfuse"] # list of success callbacks failure_callback: ["sentry"] # list of failure callbacks callbacks: ["otel"] # list of callbacks - runs on success and failure service_callbacks: ["datadog", "prometheus"] # logs redis, postgres failures on datadog, prometheus turn_off_message_logging: boolean # prevent the messages and responses from being logged to on your callbacks, but request metadata will still be logged. redact_user_api_key_info: boolean # Redact information about the user api key (hashed token, user_id, team id, etc.), from logs. Currently supported for Langfuse, OpenTelemetry, Logfire, ArizeAI logging. + langfuse_default_tags: ["cache_hit", "cache_key", "proxy_base_url", "user_api_key_alias", "user_api_key_user_id", "user_api_key_user_email", "user_api_key_team_alias", "semantic-similarity", "proxy_base_url"] # default tags for Langfuse Logging + + + set_verbose: boolean # sets litellm.set_verbose=True to view verbose debug logs. DO NOT LEAVE THIS ON IN PRODUCTION + json_logs: boolean # if true, logs will be in json format + + # Fallbacks, reliability + default_fallbacks: ["claude-opus"] # set default_fallbacks, in case a specific model group is misconfigured / bad. + content_policy_fallbacks: [{"gpt-3.5-turbo-small": ["claude-opus"]}] # fallbacks for ContentPolicyErrors + context_window_fallbacks: [{"gpt-3.5-turbo-small": ["gpt-3.5-turbo-large", "claude-opus"]}] # fallbacks for ContextWindowExceededErrors + + + + # Caching settings + cache: true + cache_params: # set cache params for redis + type: redis # type of cache to initialize + + # Optional - Redis Settings + host: "localhost" # The host address for the Redis cache. Required if type is "redis". + port: 6379 # The port number for the Redis cache. Required if type is "redis". + password: "your_password" # The password for the Redis cache. Required if type is "redis". + namespace: "litellm_caching" # namespace for redis cache + + # Optional - Redis Cluster Settings + redis_startup_nodes: [{"host": "127.0.0.1", "port": "7001"}] + + # Optional - Redis Sentinel Settings + service_name: "mymaster" + sentinel_nodes: [["localhost", 26379]] + + # Optional - Qdrant Semantic Cache Settings + qdrant_semantic_cache_embedding_model: openai-embedding # the model should be defined on the model_list + qdrant_collection_name: test_collection + qdrant_quantization_config: binary + similarity_threshold: 0.8 # similarity threshold for semantic cache + + # Optional - S3 Cache Settings + s3_bucket_name: cache-bucket-litellm # AWS Bucket Name for S3 + s3_region_name: us-west-2 # AWS Region Name for S3 + s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # us os.environ/ to pass environment variables. This is AWS Access Key ID for S3 + s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3 + s3_endpoint_url: https://s3.amazonaws.com # [OPTIONAL] S3 endpoint URL, if you want to use Backblaze/cloudflare s3 bucket + + # Common Cache settings + # Optional - Supported call types for caching + supported_call_types: ["acompletion", "atext_completion", "aembedding", "atranscription"] + # /chat/completions, /completions, /embeddings, /audio/transcriptions + mode: default_off # if default_off, you need to opt in to caching on a per call basis + ttl: 600 # ttl for caching + callback_settings: otel: @@ -646,6 +698,34 @@ general_settings: | service_callbacks | array of strings | System health monitoring - Logs redis, postgres failures on specified services (e.g. datadog, prometheus) [Doc Metrics](prometheus) | | turn_off_message_logging | boolean | If true, prevents messages and responses from being logged to callbacks, but request metadata will still be logged [Proxy Logging](logging) | | redact_user_api_key_info | boolean | If true, redacts information about the user api key from logs [Proxy Logging](logging#redacting-userapikeyinfo) | +| langfuse_default_tags | array of strings | Default tags for Langfuse Logging. Use this if you want to control which LiteLLM-specific fields are logged as tags by the LiteLLM proxy. By default LiteLLM Proxy logs no LiteLLM-specific fields as tags. [Further docs](./logging#litellm-specific-tags-on-langfuse---cache_hit-cache_key) | +| set_verbose | boolean | If true, sets litellm.set_verbose=True to view verbose debug logs. DO NOT LEAVE THIS ON IN PRODUCTION | +| json_logs | boolean | If true, logs will be in json format. If you need to store the logs as JSON, just set the `litellm.json_logs = True`. We currently just log the raw POST request from litellm as a JSON [Further docs](./debugging) | +| default_fallbacks | array of strings | List of fallback models to use if a specific model group is misconfigured / bad. [Further docs](./reliability#default-fallbacks) | +| content_policy_fallbacks | array of objects | Fallbacks to use when a ContentPolicyViolationError is encountered. [Further docs](./reliability#content-policy-fallbacks) | +| context_window_fallbacks | array of objects | Fallbacks to use when a ContextWindowExceededError is encountered. [Further docs](./reliability#context-window-fallbacks) | +| cache | boolean | If true, enables caching. [Further docs](./caching) | +| cache_params | object | Parameters for the cache. [Further docs](./caching) | +| cache_params.type | string | The type of cache to initialize. Can be one of ["local", "redis", "redis-semantic", "s3", "disk", "qdrant-semantic"]. Defaults to "redis". [Furher docs](./caching) | +| cache_params.host | string | The host address for the Redis cache. Required if type is "redis". | +| cache_params.port | integer | The port number for the Redis cache. Required if type is "redis". | +| cache_params.password | string | The password for the Redis cache. Required if type is "redis". | +| cache_params.namespace | string | The namespace for the Redis cache. | +| cache_params.redis_startup_nodes | array of objects | Redis Cluster Settings. [Further docs](./caching) | +| cache_params.service_name | string | Redis Sentinel Settings. [Further docs](./caching) | +| cache_params.sentinel_nodes | array of arrays | Redis Sentinel Settings. [Further docs](./caching) | +| cache_params.ttl | integer | The time (in seconds) to store entries in cache. | +| cache_params.qdrant_semantic_cache_embedding_model | string | The embedding model to use for qdrant semantic cache. | +| cache_params.qdrant_collection_name | string | The name of the collection to use for qdrant semantic cache. | +| cache_params.qdrant_quantization_config | string | The quantization configuration for the qdrant semantic cache. | +| cache_params.similarity_threshold | float | The similarity threshold for the semantic cache. | +| cache_params.s3_bucket_name | string | The name of the S3 bucket to use for the semantic cache. | +| cache_params.s3_region_name | string | The region name for the S3 bucket. | +| cache_params.s3_aws_access_key_id | string | The AWS access key ID for the S3 bucket. | +| cache_params.s3_aws_secret_access_key | string | The AWS secret access key for the S3 bucket. | +| cache_params.s3_endpoint_url | string | Optional - The endpoint URL for the S3 bucket. | +| cache_params.supported_call_types | array of strings | The types of calls to cache. [Further docs](./caching) | +| cache_params.mode | string | The mode of the cache. [Further docs](./caching) | ### general_settings - Reference From d742e8cb432f0f1bd613d456e1ee9fbadc2b53f0 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 8 Oct 2024 08:23:52 +0530 Subject: [PATCH 2/2] (clean up) move docker files from root to `docker` folder (#6109) * fix move docker files to docker folders * move check file length * fix docker hub deploy --- .github/workflows/ghcr_deploy.yml | 8 +++++--- check_file_length.py => ci_cd/check_file_length.py | 0 Dockerfile => docker/Dockerfile | 0 Dockerfile.alpine => docker/Dockerfile.alpine | 0 Dockerfile.custom_ui => docker/Dockerfile.custom_ui | 0 Dockerfile.database => docker/Dockerfile.database | 0 Dockerfile.non_root => docker/Dockerfile.non_root | 0 7 files changed, 5 insertions(+), 3 deletions(-) rename check_file_length.py => ci_cd/check_file_length.py (100%) rename Dockerfile => docker/Dockerfile (100%) rename Dockerfile.alpine => docker/Dockerfile.alpine (100%) rename Dockerfile.custom_ui => docker/Dockerfile.custom_ui (100%) rename Dockerfile.database => docker/Dockerfile.database (100%) rename Dockerfile.non_root => docker/Dockerfile.non_root (100%) diff --git a/.github/workflows/ghcr_deploy.yml b/.github/workflows/ghcr_deploy.yml index e3b20778e44..75015f857f6 100644 --- a/.github/workflows/ghcr_deploy.yml +++ b/.github/workflows/ghcr_deploy.yml @@ -55,6 +55,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . + file: ./docker/Dockerfile push: true tags: litellm/litellm:${{ github.event.inputs.tag || 'latest' }} - @@ -63,7 +64,7 @@ jobs: with: context: . push: true - file: Dockerfile.database + file: ./docker/Dockerfile.database tags: litellm/litellm-database:${{ github.event.inputs.tag || 'latest' }} - name: Build and push litellm-spend-logs image @@ -111,6 +112,7 @@ jobs: uses: docker/build-push-action@4976231911ebf5f32aad765192d35f942aa48cb8 with: context: . + file: ./docker/Dockerfile push: true tags: ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }} # if a tag is provided, use that, otherwise use the release tag, and if neither is available, use 'latest' labels: ${{ steps.meta.outputs.labels }} @@ -149,7 +151,7 @@ jobs: uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: context: . - file: Dockerfile.database + file: ./docker/Dockerfile.database push: true tags: ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }} labels: ${{ steps.meta-database.outputs.labels }} @@ -188,7 +190,7 @@ jobs: uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: context: . - file: Dockerfile.non_root + file: ./docker/Dockerfile.non_root push: true tags: ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.release_type }} labels: ${{ steps.meta-non_root.outputs.labels }} diff --git a/check_file_length.py b/ci_cd/check_file_length.py similarity index 100% rename from check_file_length.py rename to ci_cd/check_file_length.py diff --git a/Dockerfile b/docker/Dockerfile similarity index 100% rename from Dockerfile rename to docker/Dockerfile diff --git a/Dockerfile.alpine b/docker/Dockerfile.alpine similarity index 100% rename from Dockerfile.alpine rename to docker/Dockerfile.alpine diff --git a/Dockerfile.custom_ui b/docker/Dockerfile.custom_ui similarity index 100% rename from Dockerfile.custom_ui rename to docker/Dockerfile.custom_ui diff --git a/Dockerfile.database b/docker/Dockerfile.database similarity index 100% rename from Dockerfile.database rename to docker/Dockerfile.database diff --git a/Dockerfile.non_root b/docker/Dockerfile.non_root similarity index 100% rename from Dockerfile.non_root rename to docker/Dockerfile.non_root