From bf55602e26d801e39d44bc02c8d737a11eebe4a4 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Wed, 22 Nov 2023 22:39:34 -0500 Subject: [PATCH 1/7] (test) lambda / sam build support --- template.yaml | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 template.yaml diff --git a/template.yaml b/template.yaml new file mode 100644 index 00000000000..d179f9cfc9c --- /dev/null +++ b/template.yaml @@ -0,0 +1,104 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: > + llmlite-service + + SAM Template for llmlite-service + +# More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst +Globals: + Function: + Timeout: 600 + MemorySize: 128 + +Parameters: + AliasParameter: + Type: String + Default: live + +Resources: + LambdaExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + - edgelambda.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: LambdaExecutionPolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: 'arn:aws:logs:*:*:*' + + Function: + Type: AWS::Serverless::Function + Properties: + FunctionName: !Sub "${AWS::StackName}-llmlite" + CodeUri: "./" + Handler: litellm/proxy/lambda.lambda_handler + Runtime: python3.9 + AutoPublishAlias: !Ref AliasParameter + Role: !GetAtt LambdaExecutionRole.Arn + Architectures: + - x86_64 + DeploymentPreference: + Type: AllAtOnce + Alarms: + - !Ref NewVersionErrorMetricGreaterThanZeroAlarm + + NewVersionErrorMetricGreaterThanZeroAlarm: + Type: "AWS::CloudWatch::Alarm" + Properties: + AlarmDescription: Lambda Function Error > 0 + ComparisonOperator: GreaterThanThreshold + Dimensions: + - Name: Resource + Value: !Sub "${Function}:live" + - Name: FunctionName + Value: !Ref Function + - Name: ExecutedVersion + Value: !GetAtt Function.Version.Version + EvaluationPeriods: 1 + Unit: Count + MetricName: Errors + Namespace: AWS/Lambda + Period: 60 + Statistic: Sum + Threshold: 0 + + URL: + Type: AWS::Lambda::Url + DependsOn: FunctionAliaslive + Properties: + AuthType: AWS_IAM + Qualifier: live + TargetFunctionArn: !GetAtt Function.Arn + +Outputs: + FunctionARN: + Description: "Lambda Function ARN" + Value: !GetAtt Function.Arn + + FunctionUrl: + Description: "Lambda Function URL Endpoint" + Value: + Fn::GetAtt: URL.FunctionUrl + + FunctionVersion: + Description: "Lambda Function Version" + Value: !GetAtt Function.Version.Version + + FunctionNewAlarmARN: + Description: "Lambda Function New Alarm ARN" + Value: !GetAtt NewVersionErrorMetricGreaterThanZeroAlarm.Arn \ No newline at end of file From 65ec095c359a8981f836e20069a74c6d52982bf5 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Wed, 22 Nov 2023 22:52:36 -0500 Subject: [PATCH 2/7] (test) Bump runtime. --- template.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/template.yaml b/template.yaml index d179f9cfc9c..5b0d3b86f61 100644 --- a/template.yaml +++ b/template.yaml @@ -47,7 +47,7 @@ Resources: FunctionName: !Sub "${AWS::StackName}-llmlite" CodeUri: "./" Handler: litellm/proxy/lambda.lambda_handler - Runtime: python3.9 + Runtime: python3.11 AutoPublishAlias: !Ref AliasParameter Role: !GetAtt LambdaExecutionRole.Arn Architectures: @@ -80,7 +80,7 @@ Resources: URL: Type: AWS::Lambda::Url DependsOn: FunctionAliaslive - Properties: + Properties: AuthType: AWS_IAM Qualifier: live TargetFunctionArn: !GetAtt Function.Arn @@ -101,4 +101,4 @@ Outputs: FunctionNewAlarmARN: Description: "Lambda Function New Alarm ARN" - Value: !GetAtt NewVersionErrorMetricGreaterThanZeroAlarm.Arn \ No newline at end of file + Value: !GetAtt NewVersionErrorMetricGreaterThanZeroAlarm.Arn From 56d28ba69ec23db23e3d855e105b6dc19696dc12 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Wed, 22 Nov 2023 22:53:55 -0500 Subject: [PATCH 3/7] Don't commit .sam-aws. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1278b78670b..76d98409aba 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ litellm/proxy/api_log.json router_config.yaml litellm_server/config.yaml litellm/proxy/_secret_config.yaml +.aws-sam/ From 4d9aa7a3f71bd7feac06dbdf8067be38034d822b Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Wed, 22 Nov 2023 23:07:33 -0500 Subject: [PATCH 4/7] Add symlink and only copy in source dir to stay under 50MB compressed limit for Lambdas. --- litellm/requirements.txt | 1 + template.yaml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 120000 litellm/requirements.txt diff --git a/litellm/requirements.txt b/litellm/requirements.txt new file mode 120000 index 00000000000..dc833dd4bef --- /dev/null +++ b/litellm/requirements.txt @@ -0,0 +1 @@ +../requirements.txt \ No newline at end of file diff --git a/template.yaml b/template.yaml index 5b0d3b86f61..448cc92411d 100644 --- a/template.yaml +++ b/template.yaml @@ -45,8 +45,8 @@ Resources: Type: AWS::Serverless::Function Properties: FunctionName: !Sub "${AWS::StackName}-llmlite" - CodeUri: "./" - Handler: litellm/proxy/lambda.lambda_handler + CodeUri: "./litellm" + Handler: proxy/lambda.lambda_handler Runtime: python3.11 AutoPublishAlias: !Ref AliasParameter Role: !GetAtt LambdaExecutionRole.Arn From 60ca43b6d9e1ca9a858000bcf7155037636841ac Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Thu, 23 Nov 2023 00:04:47 -0500 Subject: [PATCH 5/7] Add mangum. --- litellm/proxy/lambda.py | 4 ++++ requirements.txt | 3 ++- template.yaml | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 litellm/proxy/lambda.py diff --git a/litellm/proxy/lambda.py b/litellm/proxy/lambda.py new file mode 100644 index 00000000000..6b278c41188 --- /dev/null +++ b/litellm/proxy/lambda.py @@ -0,0 +1,4 @@ +from mangum import Mangum +from litellm.proxy.proxy_server import app + +handler = Mangum(app, lifespan="on") diff --git a/requirements.txt b/requirements.txt index 0abf3ed2b7d..345d7ae95d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,5 @@ pyyaml rq prisma celery -psutil \ No newline at end of file +psutil +mangum \ No newline at end of file diff --git a/template.yaml b/template.yaml index 448cc92411d..baa7c2d17a5 100644 --- a/template.yaml +++ b/template.yaml @@ -46,7 +46,7 @@ Resources: Properties: FunctionName: !Sub "${AWS::StackName}-llmlite" CodeUri: "./litellm" - Handler: proxy/lambda.lambda_handler + Handler: proxy/lambda.handler Runtime: python3.11 AutoPublishAlias: !Ref AliasParameter Role: !GetAtt LambdaExecutionRole.Arn From ab09ad9a748c6c9bbc3438a57567086282e14117 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Thu, 23 Nov 2023 01:05:24 -0500 Subject: [PATCH 6/7] Make Function URL public. --- template.yaml | 38 ++++++++++++++------------------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/template.yaml b/template.yaml index baa7c2d17a5..b90d7ac261b 100644 --- a/template.yaml +++ b/template.yaml @@ -10,36 +10,27 @@ Globals: Function: Timeout: 600 MemorySize: 128 + Environment: + Variables: + WORKER_CONFIG: !Ref WorkerConfigParameter Parameters: AliasParameter: Type: String Default: live + WorkerConfigParameter: + Type: String + Description: Sample environment variable + Default: '{"model": null, "alias": null, "api_base": null, "api_version": "2023-07-01-preview", "debug": false, "temperature": null, "max_tokens": null, "request_timeout": 600, "max_budget": null, "telemetry": true, "drop_params": false, "add_function_to_prompt": false, "headers": null, "save": false, "config": null, "use_queue": false}' Resources: - LambdaExecutionRole: - Type: AWS::IAM::Role + MyUrlFunctionPermissions: + Type: AWS::Lambda::Permission Properties: - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - - edgelambda.amazonaws.com - Action: sts:AssumeRole - Policies: - - PolicyName: LambdaExecutionPolicy - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - logs:CreateLogGroup - - logs:CreateLogStream - - logs:PutLogEvents - Resource: 'arn:aws:logs:*:*:*' + FunctionName: !Ref URL + Action: lambda:InvokeFunctionUrl + Principal: "*" + FunctionUrlAuthType: NONE Function: Type: AWS::Serverless::Function @@ -49,7 +40,6 @@ Resources: Handler: proxy/lambda.handler Runtime: python3.11 AutoPublishAlias: !Ref AliasParameter - Role: !GetAtt LambdaExecutionRole.Arn Architectures: - x86_64 DeploymentPreference: @@ -81,7 +71,7 @@ Resources: Type: AWS::Lambda::Url DependsOn: FunctionAliaslive Properties: - AuthType: AWS_IAM + AuthType: NONE Qualifier: live TargetFunctionArn: !GetAtt Function.Arn From ecdb7a59e343a21c761d12ac0a2aaefd7693fad6 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Thu, 23 Nov 2023 02:09:09 -0500 Subject: [PATCH 7/7] Use -function for naming. --- template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template.yaml b/template.yaml index b90d7ac261b..d4db2c2ac1f 100644 --- a/template.yaml +++ b/template.yaml @@ -35,7 +35,7 @@ Resources: Function: Type: AWS::Serverless::Function Properties: - FunctionName: !Sub "${AWS::StackName}-llmlite" + FunctionName: !Sub "${AWS::StackName}-function" CodeUri: "./litellm" Handler: proxy/lambda.handler Runtime: python3.11