From 5a18e6b6da67f273c14fcbd47000145294ef6a40 Mon Sep 17 00:00:00 2001 From: zoveki <349028329+zoveki@users.noreply.github.com> Date: Wed, 28 Oct 2026 04:13:03 -0700 Subject: [PATCH] test(files): audit #40621 (#48788) * test(files): audit #40621 Pin the raw cloud storage file id gate on /v1/files content and delete against a running proxy: twice-encoded gs:// ids stay blocked for non-admin keys, proxy admin keys read s3:// objects, and ordinary provider ids keep working * test(files): check the full SigV4 signature on the admin raw s3 read --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../security/test_files_batches_access.py | 126 +++++++++++++++--- 1 file changed, 104 insertions(+), 22 deletions(-) diff --git a/tests/integration/security/test_files_batches_access.py b/tests/integration/security/test_files_batches_access.py index 7b6fd4652cf..119055c64c7 100644 --- a/tests/integration/security/test_files_batches_access.py +++ b/tests/integration/security/test_files_batches_access.py @@ -48,6 +48,7 @@ VERTEX_ACCESS_TOKEN: Final = "ya29.scripted-delete-token" AWS_ACCESS_KEY_ID: Final = "AKIAIOSFODNN7EXAMPLE" AWS_SECRET_ACCESS_KEY: Final = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" AWS_REGION: Final = "us-east-1" +S3_OBJECT_BODY: Final = b'{"recordId":"r1","modelOutput":{"stop_reason":"end_turn"}}\n' _ForbiddenSurface = Literal[ "upload_sdk_extra_body", "upload_form", @@ -708,6 +709,86 @@ def test_non_admin_cannot_delete_a_raw_gcs_uri_before_vertex_auth_or_storage(gat assert token_requests == (), "non-admin raw GCS delete reached the Vertex token endpoint" +def test_non_admin_still_reads_an_ordinary_provider_file_id(gateway: Gateway) -> None: + with wire_server(_openai_backend()) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="openai/gpt-4o-mini", api_base=wire.url + "/v1", api_key="provider-key-" + uuid.uuid4().hex[:8] + ) + _wait_until_every_worker_serves(gateway, model) + member: Final = _member(scenario, model) + file_id: Final = "file-" + uuid.uuid4().hex + + read: Final = gateway.request("GET", f"/v1/files/{file_id}/content", key=member.key, params={"model": model}) + assert read.status_code == 200, read.text + assert read.content == b"file-content\n", read.content + assert [ + (request.method, urlsplit(request.target).path) for request in _drained_other_than_model_list_probes(wire) + ] == [("GET", f"/v1/files/{file_id}/content")] + + +@pytest.mark.parametrize( + ("method", "route_suffix", "verb"), + [("GET", "/content", "retrieved"), ("DELETE", "", "deleted")], + ids=["content", "delete"], +) +def test_non_admin_cannot_reach_vertex_with_a_twice_encoded_raw_gcs_uri( + gateway: Gateway, method: Literal["GET", "DELETE"], route_suffix: str, verb: str +) -> None: + with ( + wire_server(_vertex_token_backend) as token_wire, + gateway.scenario() as scenario, + ): + model: Final = scenario.model( + model=VERTEX_MODEL, + api_key=None, + vertex_project=VERTEX_PROJECT, + vertex_location=VERTEX_LOCATION, + vertex_credentials=_vertex_credentials(token_wire.url), + gcs_bucket_name=BUCKET, + ) + _wait_until_every_worker_serves(gateway, model) + caller_user: Final = scenario.user(user_role="internal_user") + caller_key: Final = scenario.key(user_id=caller_user, models=[model]) + object_uri: Final = f"gs://{BUCKET}/litellm-vertex-files/litellm-vertex-files-obj.jsonl" + twice_encoded: Final = quote(quote(object_uri, safe=""), safe="") + token_wire.drain() + + denied: Final = gateway.request( + method, f"/v1/files/{twice_encoded}{route_suffix}", key=caller_key, params={"model": model} + ) + token_requests: Final = token_wire.drain() + assert denied.status_code == 403, f"{denied.text}; token requests={token_requests}" + assert _error_message(denied, 403) == ( + f"Raw cloud storage file ids can only be {verb} by a proxy admin key. " + "Use the LiteLLM managed file id returned when the file was created." + ), denied.text + assert token_requests == (), f"non-admin twice-encoded raw GCS {method} reached the Vertex token endpoint" + + +def _assert_signed_empty_body_s3_request(request: Request, method: Literal["GET", "DELETE"], wire_url: str) -> None: + assert request.method == method, request.method + request_date: Final = request.headers["x-amz-date"] + content_hash: Final = hashlib.sha256(b"").hexdigest() + assert request.headers["x-amz-content-sha256"] == content_hash, request.headers + aws_request: Final = AWSRequest( + method=method, + url=wire_url + request.target, + headers={ + "host": request.headers["host"], + "x-amz-date": request_date, + "x-amz-content-sha256": content_hash, + }, + ) + aws_request.context["timestamp"] = request_date + auth: Final = S3SigV4Auth(Credentials(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY), "s3", AWS_REGION) + signature: Final = auth.signature(auth.string_to_sign(aws_request, auth.canonical_request(aws_request)), aws_request) + expected_authorization: Final = ( + f"AWS4-HMAC-SHA256 Credential={AWS_ACCESS_KEY_ID}/{request_date[:8]}/{AWS_REGION}/s3/aws4_request, " + f"SignedHeaders={auth.signed_headers(auth.headers_to_sign(aws_request))}, Signature={signature}" + ) + assert request.headers["authorization"] == expected_authorization, request.headers["authorization"] + + def test_proxy_admin_deletes_a_raw_s3_uri_with_one_signed_delete(gateway: Gateway) -> None: with wire_server(_s3_backend()) as wire, gateway.scenario() as scenario: model: Final = _bedrock_model(scenario, wire) @@ -721,29 +802,30 @@ def test_proxy_admin_deletes_a_raw_s3_uri_with_one_signed_delete(gateway: Gatewa assert parsed.deleted is True and parsed.id == object_uri, deleted.text (request,) = _drained_other_than_model_list_probes(wire) assert request.target == f"/{BUCKET}/litellm-bedrock-files-obj.jsonl", request.target - request_date: Final = request.headers["x-amz-date"] - content_hash: Final = hashlib.sha256(b"").hexdigest() - assert request.headers["x-amz-content-sha256"] == content_hash, request.headers - aws_request: Final = AWSRequest( - method="DELETE", - url=wire.url + request.target, - headers={ - "host": request.headers["host"], - "x-amz-date": request_date, - "x-amz-content-sha256": content_hash, - }, + _assert_signed_empty_body_s3_request(request, "DELETE", wire.url) + + +def _s3_object_backend(request: Request) -> Reply: + if request.method == "GET": + return Reply(body=S3_OBJECT_BODY, content_type="application/octet-stream") + return _json_reply({"error": {"message": f"unscripted {request.method} {request.target}"}}, 404) + + +def test_proxy_admin_reads_a_raw_s3_uri_with_one_signed_get(gateway: Gateway) -> None: + with wire_server(_s3_object_backend) as wire, gateway.scenario() as scenario: + model: Final = _bedrock_model(scenario, wire) + _wait_until_every_worker_serves(gateway, model) + object_key: Final = f"litellm-batch-outputs/{uuid.uuid4().hex}.jsonl.out" + object_uri: Final = f"s3://{BUCKET}/{object_key}" + + read: Final = gateway.request( + "GET", "/v1/files/" + quote(object_uri, safe="") + "/content", params={"model": model} ) - aws_request.context["timestamp"] = request_date - auth: Final = S3SigV4Auth(Credentials(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY), "s3", AWS_REGION) - signature: Final = auth.signature( - auth.string_to_sign(aws_request, auth.canonical_request(aws_request)), aws_request - ) - expected_authorization: Final = ( - f"AWS4-HMAC-SHA256 Credential={AWS_ACCESS_KEY_ID}/{request_date[:8]}/{AWS_REGION}/s3/aws4_request, " - f"SignedHeaders={auth.signed_headers(auth.headers_to_sign(aws_request))}, Signature={signature}" - ) - assert request.headers["authorization"] == expected_authorization, request.headers["authorization"] - assert request.method == "DELETE", request.method + storage_requests: Final = _drained_other_than_model_list_probes(wire) + assert read.status_code == 200, f"{read.text}; storage requests={storage_requests}" + assert read.content == S3_OBJECT_BODY, read.content + assert [request.target for request in storage_requests] == [f"/{BUCKET}/{object_key}"], storage_requests + _assert_signed_empty_body_s3_request(storage_requests[0], "GET", wire.url) def _granting_vertex_token_backend(request: Request) -> Reply: