diff --git a/.circleci/config.yml b/.circleci/config.yml index 5f4628d26dc..d070c55dca7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -65,6 +65,7 @@ jobs: pip install "pydantic==2.7.1" pip install "diskcache==5.6.1" pip install "Pillow==10.3.0" + pip install "ijson==3.2.3" - save_cache: paths: - ./venv @@ -126,6 +127,7 @@ jobs: pip install jinja2 pip install tokenizers pip install openai + pip install ijson - run: name: Run tests command: | @@ -180,6 +182,7 @@ jobs: pip install numpydoc pip install prisma pip install fastapi + pip install ijson pip install "httpx==0.24.1" pip install "gunicorn==21.2.0" pip install "anyio==3.7.1" @@ -202,6 +205,7 @@ jobs: -e REDIS_PORT=$REDIS_PORT \ -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \ -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \ + -e MISTRAL_API_KEY=$MISTRAL_API_KEY \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 00000000000..58e7cfe10da --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + groups: + github-actions: + patterns: + - "*" diff --git a/.github/workflows/ghcr_deploy.yml b/.github/workflows/ghcr_deploy.yml index 58cda02c31b..4045056e3a2 100644 --- a/.github/workflows/ghcr_deploy.yml +++ b/.github/workflows/ghcr_deploy.yml @@ -25,6 +25,11 @@ jobs: if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest steps: + - + name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.commit_hash }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -41,12 +46,14 @@ jobs: name: Build and push uses: docker/build-push-action@v5 with: + context: . push: true tags: litellm/litellm:${{ github.event.inputs.tag || 'latest' }} - name: Build and push litellm-database image uses: docker/build-push-action@v5 with: + context: . push: true file: Dockerfile.database tags: litellm/litellm-database:${{ github.event.inputs.tag || 'latest' }} @@ -54,6 +61,7 @@ jobs: name: Build and push litellm-spend-logs image uses: docker/build-push-action@v5 with: + context: . push: true file: ./litellm-js/spend-logs/Dockerfile tags: litellm/litellm-spend_logs:${{ github.event.inputs.tag || 'latest' }} @@ -68,6 +76,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.commit_hash }} # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - name: Log in to the Container registry uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 @@ -92,7 +102,7 @@ jobs: - name: Build and push Docker image uses: docker/build-push-action@4976231911ebf5f32aad765192d35f942aa48cb8 with: - context: https://github.com/BerriAI/litellm.git#${{ github.event.inputs.commit_hash}} + context: . push: true tags: ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }} # if a tag is provided, use that, otherwise use the release tag, and if neither is available, use 'latest' labels: ${{ steps.meta.outputs.labels }} @@ -106,6 +116,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.commit_hash }} - name: Log in to the Container registry uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 @@ -128,7 +140,7 @@ jobs: - name: Build and push Database Docker image uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: - context: https://github.com/BerriAI/litellm.git#${{ github.event.inputs.commit_hash}} + context: . file: Dockerfile.database push: true tags: ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }} @@ -143,6 +155,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.commit_hash }} - name: Log in to the Container registry uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 @@ -165,7 +179,7 @@ jobs: - name: Build and push Database Docker image uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: - context: https://github.com/BerriAI/litellm.git#${{ github.event.inputs.commit_hash}} + context: . file: ./litellm-js/spend-logs/Dockerfile push: true tags: ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }} @@ -176,6 +190,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.commit_hash }} - name: Log in to the Container registry uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cc41d85f145..9af3f2e8aac 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,19 @@ repos: +- repo: local + hooks: + - id: mypy + name: mypy + entry: python3 -m mypy --ignore-missing-imports + language: system + types: [python] + files: ^litellm/ + - id: isort + name: isort + entry: isort + language: system + types: [python] + files: litellm/.*\.py + exclude: ^litellm/__init__.py$ - repo: https://github.com/psf/black rev: 24.2.0 hooks: @@ -16,11 +31,10 @@ repos: name: Check if files match entry: python3 ci_cd/check_files_match.py language: system -- repo: local - hooks: - - id: mypy - name: mypy - entry: python3 -m mypy --ignore-missing-imports - language: system - types: [python] - files: ^litellm/ \ No newline at end of file + # - id: check-file-length + # name: Check file length + # entry: python check_file_length.py + # args: ["10000"] # set your desired maximum number of lines + # language: python + # files: litellm/.*\.py + # exclude: ^litellm/tests/ \ No newline at end of file diff --git a/check_file_length.py b/check_file_length.py new file mode 100644 index 00000000000..f23b79add25 --- /dev/null +++ b/check_file_length.py @@ -0,0 +1,28 @@ +import sys + + +def check_file_length(max_lines, filenames): + bad_files = [] + for filename in filenames: + with open(filename, "r") as file: + lines = file.readlines() + if len(lines) > max_lines: + bad_files.append((filename, len(lines))) + return bad_files + + +if __name__ == "__main__": + max_lines = int(sys.argv[1]) + filenames = sys.argv[2:] + + bad_files = check_file_length(max_lines, filenames) + if bad_files: + bad_files.sort( + key=lambda x: x[1], reverse=True + ) # Sort files by length in descending order + for filename, length in bad_files: + print(f"{filename}: {length} lines") + + sys.exit(1) + else: + sys.exit(0) diff --git a/docs/my-website/docs/completion/input.md b/docs/my-website/docs/completion/input.md index 6ad412af8f7..e9ea8f50ef9 100644 --- a/docs/my-website/docs/completion/input.md +++ b/docs/my-website/docs/completion/input.md @@ -162,7 +162,7 @@ def completion( - `function`: *object* - Required. -- `tool_choice`: *string or object (optional)* - Controls which (if any) function is called by the model. none means the model will not call a function and instead generates a message. auto means the model can pick between generating a message or calling a function. Specifying a particular function via {"type: "function", "function": {"name": "my_function"}} forces the model to call that function. +- `tool_choice`: *string or object (optional)* - Controls which (if any) function is called by the model. none means the model will not call a function and instead generates a message. auto means the model can pick between generating a message or calling a function. Specifying a particular function via `{"type: "function", "function": {"name": "my_function"}}` forces the model to call that function. - `none` is the default when no functions are present. `auto` is the default if functions are present. diff --git a/docs/my-website/docs/debugging/hosted_debugging.md b/docs/my-website/docs/debugging/hosted_debugging.md index 5c98ac6f568..e69de29bb2d 100644 --- a/docs/my-website/docs/debugging/hosted_debugging.md +++ b/docs/my-website/docs/debugging/hosted_debugging.md @@ -1,90 +0,0 @@ -import Image from '@theme/IdealImage'; -import QueryParamReader from '../../src/components/queryParamReader.js' - -# [Beta] Monitor Logs in Production - -:::note - -This is in beta. Expect frequent updates, as we improve based on your feedback. - -::: - -LiteLLM provides an integration to let you monitor logs in production. - -👉 Jump to our sample LiteLLM Dashboard: https://admin.litellm.ai/ - - -Dashboard - -## Debug your first logs - - Open In Colab - - - -### 1. Get your LiteLLM Token - -Go to [admin.litellm.ai](https://admin.litellm.ai/) and copy the code snippet with your unique token - -Usage - -### 2. Set up your environment - -**Add it to your .env** - -```python -import os - -os.env["LITELLM_TOKEN"] = "e24c4c06-d027-4c30-9e78-18bc3a50aebb" # replace with your unique token - -``` - -**Turn on LiteLLM Client** -```python -import litellm -litellm.client = True -``` - -### 3. Make a normal `completion()` call -```python -import litellm -from litellm import completion -import os - -# set env variables -os.environ["LITELLM_TOKEN"] = "e24c4c06-d027-4c30-9e78-18bc3a50aebb" # replace with your unique token -os.environ["OPENAI_API_KEY"] = "openai key" - -litellm.use_client = True # enable logging dashboard -messages = [{ "content": "Hello, how are you?","role": "user"}] - -# openai call -response = completion(model="gpt-3.5-turbo", messages=messages) -``` - -Your `completion()` call print with a link to your session dashboard (https://admin.litellm.ai/) - -In the above case it would be: [`admin.litellm.ai/e24c4c06-d027-4c30-9e78-18bc3a50aebb`](https://admin.litellm.ai/e24c4c06-d027-4c30-9e78-18bc3a50aebb) - -Click on your personal dashboard link. Here's how you can find it 👇 - -Dashboard - -[👋 Tell us if you need better privacy controls](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version?month=2023-08) - -### 3. Review request log - -Oh! Looks like our request was made successfully. Let's click on it and see exactly what got sent to the LLM provider. - - - - -Ah! So we can see that this request was made to a **Baseten** (see litellm_params > custom_llm_provider) for a model with ID - **7qQNLDB** (see model). The message sent was - `"Hey, how's it going?"` and the response received was - `"As an AI language model, I don't have feelings or emotions, but I can assist you with your queries. How can I assist you today?"` - -Dashboard Log Row - -:::info - -🎉 Congratulations! You've successfully debugger your first log! - -::: \ No newline at end of file diff --git a/docs/my-website/docs/observability/athina_integration.md b/docs/my-website/docs/observability/athina_integration.md index 62c88975185..cd1442f35ab 100644 --- a/docs/my-website/docs/observability/athina_integration.md +++ b/docs/my-website/docs/observability/athina_integration.md @@ -2,6 +2,15 @@ import Image from '@theme/IdealImage'; # Athina + +:::tip + +This is community maintained, Please make an issue if you run into a bug +https://github.com/BerriAI/litellm + +::: + + [Athina](https://athina.ai/) is an evaluation framework and production monitoring platform for your LLM-powered app. Athina is designed to enhance the performance and reliability of AI applications through real-time monitoring, granular analytics, and plug-and-play evaluations. diff --git a/docs/my-website/docs/observability/greenscale_integration.md b/docs/my-website/docs/observability/greenscale_integration.md index 0dd673226e1..49eadc6453d 100644 --- a/docs/my-website/docs/observability/greenscale_integration.md +++ b/docs/my-website/docs/observability/greenscale_integration.md @@ -1,5 +1,14 @@ # Greenscale - Track LLM Spend and Responsible Usage + +:::tip + +This is community maintained, Please make an issue if you run into a bug +https://github.com/BerriAI/litellm + +::: + + [Greenscale](https://greenscale.ai/) is a production monitoring platform for your LLM-powered app that provides you granular key insights into your GenAI spending and responsible usage. Greenscale only captures metadata to minimize the exposure risk of personally identifiable information (PII). ## Getting Started diff --git a/docs/my-website/docs/observability/helicone_integration.md b/docs/my-website/docs/observability/helicone_integration.md index de89ba8da8c..f7fd330c309 100644 --- a/docs/my-website/docs/observability/helicone_integration.md +++ b/docs/my-website/docs/observability/helicone_integration.md @@ -1,4 +1,13 @@ # Helicone Tutorial + +:::tip + +This is community maintained, Please make an issue if you run into a bug +https://github.com/BerriAI/litellm + +::: + + [Helicone](https://helicone.ai/) is an open source observability platform that proxies your OpenAI traffic and provides you key insights into your spend, latency and usage. ## Use Helicone to log requests across all LLM Providers (OpenAI, Azure, Anthropic, Cohere, Replicate, PaLM) diff --git a/docs/my-website/docs/observability/langfuse_integration.md b/docs/my-website/docs/observability/langfuse_integration.md index 07970f599d8..9703d38a03b 100644 --- a/docs/my-website/docs/observability/langfuse_integration.md +++ b/docs/my-website/docs/observability/langfuse_integration.md @@ -1,6 +1,6 @@ import Image from '@theme/IdealImage'; -# Langfuse - Logging LLM Input/Output +# 🔥 Langfuse - Logging LLM Input/Output LangFuse is open Source Observability & Analytics for LLM Apps Detailed production traces and a granular view on quality, cost and latency @@ -122,10 +122,12 @@ response = completion( metadata={ "generation_name": "ishaan-test-generation", # set langfuse Generation Name "generation_id": "gen-id22", # set langfuse Generation ID + "parent_observation_id": "obs-id9" # set langfuse Parent Observation ID "version": "test-generation-version" # set langfuse Generation Version "trace_user_id": "user-id2", # set langfuse Trace User ID "session_id": "session-1", # set langfuse Session ID "tags": ["tag1", "tag2"], # set langfuse Tags + "trace_name": "new-trace-name" # set langfuse Trace Name "trace_id": "trace-id22", # set langfuse Trace ID "trace_metadata": {"key": "value"}, # set langfuse Trace Metadata "trace_version": "test-trace-version", # set langfuse Trace Version (if not set, defaults to Generation Version) @@ -147,9 +149,10 @@ print(response) You can also pass `metadata` as part of the request header with a `langfuse_*` prefix: ```shell -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Content-Type: application/json' \ - --header 'langfuse_trace_id: trace-id22' \ +curl --location --request POST 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'langfuse_trace_id: trace-id2' \ --header 'langfuse_trace_user_id: user-id2' \ --header 'langfuse_trace_metadata: {"key":"value"}' \ --data '{ @@ -190,9 +193,10 @@ The following parameters can be updated on a continuation of a trace by passing #### Generation Specific Parameters -* `generation_id` - Identifier for the generation, auto-generated by default -* `generation_name` - Identifier for the generation, auto-generated by default -* `prompt` - Langfuse prompt object used for the generation, defaults to None +* `generation_id` - Identifier for the generation, auto-generated by default +* `generation_name` - Identifier for the generation, auto-generated by default +* `parent_observation_id` - Identifier for the parent observation, defaults to `None` +* `prompt` - Langfuse prompt object used for the generation, defaults to `None` Any other key value pairs passed into the metadata not listed in the above spec for a `litellm` completion will be added as a metadata key value pair for the generation. diff --git a/docs/my-website/docs/observability/langsmith_integration.md b/docs/my-website/docs/observability/langsmith_integration.md index b115866d545..c038abd8214 100644 --- a/docs/my-website/docs/observability/langsmith_integration.md +++ b/docs/my-website/docs/observability/langsmith_integration.md @@ -1,6 +1,16 @@ import Image from '@theme/IdealImage'; # Langsmith - Logging LLM Input/Output + + +:::tip + +This is community maintained, Please make an issue if you run into a bug +https://github.com/BerriAI/litellm + +::: + + An all-in-one developer platform for every step of the application lifecycle https://smith.langchain.com/ diff --git a/docs/my-website/docs/observability/logfire_integration.md b/docs/my-website/docs/observability/logfire_integration.md index c1f425f4251..a2d406f9c63 100644 --- a/docs/my-website/docs/observability/logfire_integration.md +++ b/docs/my-website/docs/observability/logfire_integration.md @@ -1,6 +1,6 @@ import Image from '@theme/IdealImage'; -# Logfire - Logging LLM Input/Output +# 🔥 Logfire - Logging LLM Input/Output Logfire is open Source Observability & Analytics for LLM Apps Detailed production traces and a granular view on quality, cost and latency @@ -14,10 +14,14 @@ join our [discord](https://discord.gg/wuPM9dRgDw) ## Pre-Requisites -Ensure you have run `pip install logfire` for this integration +Ensure you have installed the following packages to use this integration ```shell -pip install logfire litellm +pip install litellm + +pip install opentelemetry-api==1.25.0 +pip install opentelemetry-sdk==1.25.0 +pip install opentelemetry-exporter-otlp==1.25.0 ``` ## Quick Start @@ -25,8 +29,7 @@ pip install logfire litellm Get your Logfire token from [Logfire](https://logfire.pydantic.dev/) ```python -litellm.success_callback = ["logfire"] -litellm.failure_callback = ["logfire"] # logs errors to logfire +litellm.callbacks = ["logfire"] ``` ```python diff --git a/docs/my-website/docs/observability/lunary_integration.md b/docs/my-website/docs/observability/lunary_integration.md index 9b8e90df7b4..56e74132f78 100644 --- a/docs/my-website/docs/observability/lunary_integration.md +++ b/docs/my-website/docs/observability/lunary_integration.md @@ -1,5 +1,13 @@ # Lunary - Logging and tracing LLM input/output +:::tip + +This is community maintained, Please make an issue if you run into a bug +https://github.com/BerriAI/litellm + +::: + + [Lunary](https://lunary.ai/) is an open-source AI developer platform providing observability, prompt management, and evaluation tools for AI developers.