diff --git a/.github/workflows/ghcr_deploy.yml b/.github/workflows/ghcr_deploy.yml index aa032972b80..f67538a4272 100644 --- a/.github/workflows/ghcr_deploy.yml +++ b/.github/workflows/ghcr_deploy.yml @@ -320,72 +320,36 @@ jobs: run: | echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV} - - name: Get LiteLLM Latest Tag - id: current_app_tag - shell: bash - run: | - LATEST_TAG=$(git describe --tags --exclude "*dev*" --abbrev=0) - if [ -z "${LATEST_TAG}" ]; then - echo "latest_tag=latest" | tee -a $GITHUB_OUTPUT - else - echo "latest_tag=${LATEST_TAG}" | tee -a $GITHUB_OUTPUT - fi - - - name: Get last published chart version - id: current_version - shell: bash - run: | - CHART_LIST=$(helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/${{ env.CHART_NAME }} 2>/dev/null || true) - if [ -z "${CHART_LIST}" ]; then - echo "current-version=1.0.0" | tee -a $GITHUB_OUTPUT - else - # Extract version and strip any prerelease suffix (e.g., 1.0.5-latest -> 1.0.5) - VERSION=$(printf '%s' "${CHART_LIST}" | grep '^version:' | awk 'BEGIN{FS=":"}{print $2}' | tr -d " " | cut -d'-' -f1) - echo "current-version=${VERSION}" | tee -a $GITHUB_OUTPUT - fi - env: - HELM_EXPERIMENTAL_OCI: '1' - - # Automatically update the helm chart version one "patch" level - - name: Bump release version - id: bump_version - uses: christian-draeger/increment-semantic-version@1.1.0 - with: - current-version: ${{ steps.current_version.outputs.current-version || '1.0.0' }} - version-fragment: 'bug' - - # Add suffix for non-stable releases (semantic versioning) + # Sync Helm chart version with LiteLLM release version (1-1 versioning) + # This allows users to easily map Helm chart versions to LiteLLM versions + # See: https://codefresh.io/docs/docs/ci-cd-guides/helm-best-practices/ - name: Calculate chart and app versions id: chart_version shell: bash run: | - BASE_VERSION="${{ steps.bump_version.outputs.next-version || '1.0.0' }}" - RELEASE_TYPE="${{ github.event.inputs.release_type }}" INPUT_TAG="${{ github.event.inputs.tag }}" + RELEASE_TYPE="${{ github.event.inputs.release_type }}" - # Chart version (independent Helm chart versioning with release type suffix) - if [ "$RELEASE_TYPE" = "stable" ]; then - echo "version=${BASE_VERSION}" | tee -a $GITHUB_OUTPUT - else - echo "version=${BASE_VERSION}-${RELEASE_TYPE}" | tee -a $GITHUB_OUTPUT + # Chart version = LiteLLM version without 'v' prefix (Helm semver convention) + # v1.81.0 -> 1.81.0, v1.81.0.rc.1 -> 1.81.0.rc.1 + CHART_VERSION="${INPUT_TAG#v}" + + # Add suffix for 'latest' releases (rc already has suffix in tag) + if [ "$RELEASE_TYPE" = "latest" ]; then + CHART_VERSION="${CHART_VERSION}-latest" fi - # App version (must match Docker tags) - # stable/rc releases: Docker creates main-{tag}, so use the tag - # latest/dev releases: Docker only creates main-{release_type}, so use release_type - if [ "$RELEASE_TYPE" = "stable" ] || [ "$RELEASE_TYPE" = "rc" ]; then - APP_VERSION="${INPUT_TAG}" - else - APP_VERSION="${RELEASE_TYPE}" - fi + # App version = Docker tag (keeps 'v' prefix to match Docker image tags) + APP_VERSION="${INPUT_TAG}" + echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT - uses: ./.github/actions/helm-oci-chart-releaser with: name: ${{ env.CHART_NAME }} repository: ${{ env.REPO_OWNER }} - tag: ${{ github.event.inputs.chartVersion || steps.chart_version.outputs.version || '1.0.0' }} + tag: ${{ steps.chart_version.outputs.version }} app_version: ${{ steps.chart_version.outputs.app_version }} path: deploy/charts/${{ env.CHART_NAME }} registry: ${{ env.REGISTRY }} diff --git a/.github/workflows/ghcr_helm_deploy.yml b/.github/workflows/ghcr_helm_deploy.yml index f78dc6f0f3f..21b2eaafe19 100644 --- a/.github/workflows/ghcr_helm_deploy.yml +++ b/.github/workflows/ghcr_helm_deploy.yml @@ -1,10 +1,12 @@ -# this workflow is triggered by an API call when there is a new PyPI release of LiteLLM +# Standalone workflow to publish LiteLLM Helm Chart +# Note: The main ghcr_deploy.yml workflow also publishes the Helm chart as part of a full release name: Build, Publish LiteLLM Helm Chart. New Release on: workflow_dispatch: inputs: - chartVersion: - description: "Update the helm chart's version to this" + tag: + description: "LiteLLM version tag (e.g., v1.81.0)" + required: true # Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds. env: @@ -31,24 +33,22 @@ jobs: run: | echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV} - - name: Get LiteLLM Latest Tag - id: current_app_tag - uses: WyriHaximus/github-action-get-previous-tag@v1.3.0 - - - name: Get last published chart version - id: current_version + # Sync Helm chart version with LiteLLM release version (1-1 versioning) + - name: Calculate chart and app versions + id: chart_version shell: bash - run: helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/litellm-helm | grep '^version:' | awk 'BEGIN{FS=":"}{print "current-version="$2}' | tr -d " " | tee -a $GITHUB_OUTPUT - env: - HELM_EXPERIMENTAL_OCI: '1' + run: | + INPUT_TAG="${{ github.event.inputs.tag }}" - # Automatically update the helm chart version one "patch" level - - name: Bump release version - id: bump_version - uses: christian-draeger/increment-semantic-version@1.1.0 - with: - current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }} - version-fragment: 'bug' + # Chart version = LiteLLM version without 'v' prefix + # v1.81.0 -> 1.81.0 + CHART_VERSION="${INPUT_TAG#v}" + + # App version = Docker tag (keeps 'v' prefix) + APP_VERSION="${INPUT_TAG}" + + echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT + echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT - name: Lint helm chart run: helm lint deploy/charts/litellm-helm @@ -57,8 +57,8 @@ jobs: with: name: litellm-helm repository: ${{ env.REPO_OWNER }} - tag: ${{ github.event.inputs.chartVersion || steps.bump_version.outputs.next-version || '0.1.0' }} - app_version: ${{ steps.current_app_tag.outputs.tag || 'latest' }} + tag: ${{ steps.chart_version.outputs.version }} + app_version: ${{ steps.chart_version.outputs.app_version }} path: deploy/charts/litellm-helm registry: ${{ env.REGISTRY }} registry_username: ${{ github.actor }} diff --git a/docker/prod_entrypoint.sh b/docker/prod_entrypoint.sh index 1fc09d2c864..28d1bdcc294 100644 --- a/docker/prod_entrypoint.sh +++ b/docker/prod_entrypoint.sh @@ -2,6 +2,7 @@ if [ "$SEPARATE_HEALTH_APP" = "1" ]; then export LITELLM_ARGS="$@" + export SUPERVISORD_STOPWAITSECS="${SUPERVISORD_STOPWAITSECS:-3600}" exec supervisord -c /etc/supervisord.conf fi diff --git a/docker/supervisord.conf b/docker/supervisord.conf index 877335804fe..ba9d99d18a5 100644 --- a/docker/supervisord.conf +++ b/docker/supervisord.conf @@ -16,6 +16,7 @@ priority=1 exitcodes=0 stopasgroup=true killasgroup=true +stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s stdout_logfile=/dev/stdout stderr_logfile=/dev/stderr stdout_logfile_maxbytes = 0 @@ -31,6 +32,7 @@ priority=2 exitcodes=0 stopasgroup=true killasgroup=true +stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s stdout_logfile=/dev/stdout stderr_logfile=/dev/stderr stdout_logfile_maxbytes = 0 diff --git a/docs/my-website/docs/providers/gemini.md b/docs/my-website/docs/providers/gemini.md index 32dea2069b7..110e3f3f090 100644 --- a/docs/my-website/docs/providers/gemini.md +++ b/docs/my-website/docs/providers/gemini.md @@ -15,6 +15,17 @@ import TabItem from '@theme/TabItem';
+:::tip Gemini API vs Vertex AI +| Model Format | Provider | Auth Required | +|-------------|----------|---------------| +| `gemini/gemini-2.0-flash` | Gemini API | `GEMINI_API_KEY` (simple API key) | +| `vertex_ai/gemini-2.0-flash` | Vertex AI | GCP credentials + project | +| `gemini-2.0-flash` (no prefix) | Vertex AI | GCP credentials + project | + +**If you just want to use an API key** (like OpenAI), use the `gemini/` prefix. + +Models without a prefix default to Vertex AI which requires full GCP authentication. +::: ## API Keys diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index be2bf86ab10..5647b5292ef 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -14,6 +14,17 @@ import TabItem from '@theme/TabItem'; | Base URL | 1. Regional endpoints
`https://{vertex_location}-aiplatform.googleapis.com/`
2. Global endpoints (limited availability)
`https://aiplatform.googleapis.com/`| | Supported Operations | [`/chat/completions`](#sample-usage), `/completions`, [`/embeddings`](#embedding-models), [`/audio/speech`](#text-to-speech-apis), [`/fine_tuning`](#fine-tuning-apis), [`/batches`](#batch-apis), [`/files`](#batch-apis), [`/images`](#image-generation-models), [`/rerank`](#rerank-api) | +:::tip Vertex AI vs Gemini API +| Model Format | Provider | Auth Required | +|-------------|----------|---------------| +| `vertex_ai/gemini-2.0-flash` | Vertex AI | GCP credentials + project | +| `gemini-2.0-flash` (no prefix) | Vertex AI | GCP credentials + project | +| `gemini/gemini-2.0-flash` | Gemini API | `GEMINI_API_KEY` (simple API key) | + +**If you just want to use an API key** (like OpenAI), use the `gemini/` prefix instead. See [Gemini - Google AI Studio](./gemini.md). + +Models without a prefix default to Vertex AI which requires GCP authentication. +:::

diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 53b0f3eea71..f76fd214682 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -875,6 +875,7 @@ router_settings: | SECRET_MANAGER_REFRESH_INTERVAL | Refresh interval in seconds for secret manager. Default is 86400 (24 hours) | SEPARATE_HEALTH_APP | If set to '1', runs health endpoints on a separate ASGI app and port. Default: '0'. | SEPARATE_HEALTH_PORT | Port for the separate health endpoints app. Only used if SEPARATE_HEALTH_APP=1. Default: 4001. +| SUPERVISORD_STOPWAITSECS | Upper bound timeout in seconds for graceful shutdown when SEPARATE_HEALTH_APP=1. Default: 3600 (1 hour). | SERVER_ROOT_PATH | Root path for the server application | SEND_USER_API_KEY_ALIAS | Flag to send user API key alias to Zscaler AI Guard. Default is False | SEND_USER_API_KEY_TEAM_ID | Flag to send user API key team ID to Zscaler AI Guard. Default is False diff --git a/docs/my-website/docs/proxy/prod.md b/docs/my-website/docs/proxy/prod.md index 9216b0fbf30..a42d91a7d5f 100644 --- a/docs/my-website/docs/proxy/prod.md +++ b/docs/my-website/docs/proxy/prod.md @@ -277,8 +277,13 @@ Set the following environment variable(s): ```bash SEPARATE_HEALTH_APP="1" # Default "0" SEPARATE_HEALTH_PORT="8001" # Default "4001", Works only if `SEPARATE_HEALTH_APP` is "1" +SUPERVISORD_STOPWAITSECS="3600" # Optional: Upper bound timeout in seconds for graceful shutdown. Default: 3600 (1 hour). Only used when SEPARATE_HEALTH_APP=1. ``` +**Graceful Shutdown:** + +Previously, `stopwaitsecs` was not set, defaulting to 10 seconds and causing in-flight requests to fail. `SUPERVISORD_STOPWAITSECS` (default: 3600) provides an upper bound for graceful shutdown, allowing uvicorn to wait for all in-flight requests to complete. +