From 30724d9e51cdc2c3e0eb063271b4f171bc01b382 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Thu, 21 Sep 2023 16:16:16 -0700 Subject: [PATCH] sagemaker streaming --- .../docs/providers/aws_sagemaker.md | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/my-website/docs/providers/aws_sagemaker.md b/docs/my-website/docs/providers/aws_sagemaker.md index 4c9c4f1ece7..2230b9f2977 100644 --- a/docs/my-website/docs/providers/aws_sagemaker.md +++ b/docs/my-website/docs/providers/aws_sagemaker.md @@ -27,6 +27,28 @@ response = completion( ) ``` +### Usage - Streaming +Sagemaker currently does not support streaming - LiteLLM fakes streaming by returning chunks of the response string + +```python +import os +from litellm import completion + +os.environ["AWS_ACCESS_KEY_ID"] = "" +os.environ["AWS_SECRET_ACCESS_KEY"] = "" +os.environ["AWS_REGION_NAME"] = "" + +response = completion( + model="sagemaker/jumpstart-dft-meta-textgeneration-llama-2-7b", + messages=[{ "content": "Hello, how are you?","role": "user"}], + temperature=0.2, + max_tokens=80, + stream=True, + ) +for chunk in response: + print(chunk) +``` + ### AWS Sagemaker Models Here's an example of using a sagemaker model with LiteLLM