mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-06 02:48:13 +00:00
fix(vertex_ai): strip the batch custom_id tag via excludedFields, not keyField
Probed live: instanceConfig.keyField does not remove the field from the instances the container receives, so vLLM 400s every row; excludedFields performs the strip and attaches the value to the output row.
This commit is contained in:
parent
7c35312657
commit
783978b39c
3 changed files with 7 additions and 5 deletions
|
|
@ -396,12 +396,13 @@ class VertexAIBatchPrediction(VertexLLM):
|
|||
"outputConfig": vertex_batch_request["outputConfig"],
|
||||
"unmanagedContainerModel": unmanaged,
|
||||
"dedicatedResources": batch_resources,
|
||||
# keyField strips the custom_id tag from each instance before it reaches the
|
||||
# container (vLLM rejects unknown fields) and echoes it back as `key` in the output
|
||||
# row; it only takes effect alongside an explicit instanceType.
|
||||
# excludedFields strips the custom_id tag from each instance before it reaches the
|
||||
# container (vLLM rejects unknown fields) and attaches it to the output row's
|
||||
# instance echo; keyField does NOT strip (probed live: the container still received
|
||||
# the tag and 400'd every row).
|
||||
"instanceConfig": {
|
||||
"instanceType": "object",
|
||||
"keyField": VERTEX_CUSTOM_ENDPOINT_KEY_FIELD,
|
||||
"excludedFields": [VERTEX_CUSTOM_ENDPOINT_KEY_FIELD],
|
||||
},
|
||||
}
|
||||
return resolved
|
||||
|
|
|
|||
|
|
@ -700,6 +700,7 @@ class UnmanagedContainerModel(TypedDict, total=False):
|
|||
class BatchInstanceConfig(TypedDict, total=False):
|
||||
instanceType: ReadOnly[str]
|
||||
keyField: ReadOnly[str]
|
||||
excludedFields: ReadOnly[list[str]]
|
||||
|
||||
|
||||
class VertexAIBatchPredictionJob(TypedDict, total=False):
|
||||
|
|
|
|||
|
|
@ -342,7 +342,7 @@ def test_create_batch_sync_custom_endpoint_builds_unmanaged_container_job():
|
|||
"startingReplicaCount": 1,
|
||||
"maxReplicaCount": 2,
|
||||
}
|
||||
assert sent["instanceConfig"] == {"instanceType": "object", "keyField": "litellm_custom_id"}
|
||||
assert sent["instanceConfig"] == {"instanceType": "object", "excludedFields": ["litellm_custom_id"]}
|
||||
|
||||
|
||||
def test_create_batch_sync_custom_endpoint_without_container_spec_raises_400():
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue