diff --git a/terraform/provider/CHANGELOG.md b/terraform/provider/CHANGELOG.md index 4a1838a7b60..757089771d3 100644 --- a/terraform/provider/CHANGELOG.md +++ b/terraform/provider/CHANGELOG.md @@ -51,6 +51,7 @@ longer signal it. ### Fixed +- **fallback**: `litellm_fallback` create and update no longer fail with "Provider produced inconsistent result after apply: root object was present, but now absent" on a proxy with more than one replica (#74, #77). The read-back right after `POST /fallback` could reach a replica that had not reloaded the write yet (replicas reload fallbacks every `proxy_config_reload_interval_seconds`, 30 s by default), which answered 404 or the previous `fallback_models`. The provider now retries that read for up to 45 s, the way `litellm_model` retries its read after create. A refresh still treats 404 as removed, so drift is still detected - **model**: `additional_drop_params` in `litellm_model.additional_litellm_params` is now sent to the proxy as the deployment's `additional_drop_params` list, the same list a `model_list` entry takes. Before, the provider never sent the list and instead deleted every key it named from `litellm_params`, so a `store` or `vertex_location` listed there was lost and requests still carried the parameters (#38349). The value must be a JSON list of parameter names such as `jsonencode(["store"])`, which `terraform plan` now checks, and `null` entries are skipped as before. When the configuration sets the list, refresh reads the stored list back from the proxy, so a list changed outside Terraform shows up as drift. On a model an older provider created, the proxy has no stored list, so the first plan after upgrading shows the list and every listed key the configuration sets as changes, since that apply writes them. A write-only secret such as `model_api_key_wo` is never shown in a plan - **user**: `litellm_user` updates no longer send `blocked`, which `/user/update` writes into the user table where no such column exists, so every update failed with a 400 (`Could not find field at upsertOneLiteLLM_UserTable.create.blocked`). Create still sends `blocked`, including `false`, so a `default_internal_user_params.blocked` on the proxy cannot block the auto-created key. Changing `blocked` after creation has no effect, since the proxy has no user-level block; use `litellm_key_block` instead - **model**: `litellm_model` refresh now reads the `{"data": [...]}` envelope `/model/info` returns, so `model_info` fields changed outside Terraform show up as drift instead of silently keeping the previous state diff --git a/terraform/provider/litellm/resource_fallback.go b/terraform/provider/litellm/resource_fallback.go index 680e051e60e..25b11ddadd4 100644 --- a/terraform/provider/litellm/resource_fallback.go +++ b/terraform/provider/litellm/resource_fallback.go @@ -6,6 +6,8 @@ import ( "log" "net/http" "net/url" + "reflect" + "time" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" @@ -13,6 +15,15 @@ import ( const endpointFallbackCreate = "/fallback" +// fallbackReadBackDelays are the waits between read-backs after a fallback write. GET /fallback/{model} +// answers from the in-memory router of whichever replica serves it, and replicas reload fallbacks from +// the database only every proxy_config_reload_interval_seconds (30 s by default), so on a multi-replica +// proxy the first read-back often reaches a replica that has not loaded the write yet. The waits add up +// to 45 s, longer than one default reload. +var fallbackReadBackDelays = []time.Duration{ + 1 * time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second, 10 * time.Second, 10 * time.Second, 10 * time.Second, +} + type FallbackGetResponse struct { Model string `json:"model"` FallbackModels []string `json:"fallback_models"` @@ -85,7 +96,44 @@ func upsertLiteLLMFallback(d *schema.ResourceData, m interface{}, action string) } d.SetId(d.Get("model").(string)) - return resourceLiteLLMFallbackRead(d, m) + return readFallbackAfterWrite(d, m) +} + +// readFallbackAfterWrite reads a fallback back after POST /fallback, retrying while the proxy answers 404 +// or still returns the previous fallback_models, since the read can reach a replica that has not reloaded +// the write yet (see fallbackReadBackDelays). A refresh keeps treating 404 as removed, through +// resourceLiteLLMFallbackRead, so drift is still detected. +func readFallbackAfterWrite(d *schema.ResourceData, m interface{}) error { + id := d.Id() + want := expandStringList(d.Get("fallback_models").([]interface{})) + + for attempt := 0; ; attempt++ { + if err := resourceLiteLLMFallbackRead(d, m); err != nil { + return err + } + got := expandStringList(d.Get("fallback_models").([]interface{})) + if d.Id() != "" && reflect.DeepEqual(got, want) { + return nil + } + if attempt == len(fallbackReadBackDelays) { + break + } + delay := fallbackReadBackDelays[attempt] + if d.Id() == "" { + log.Printf("[INFO] Fallback for model %s not visible yet (replica not reloaded), retrying in %v", id, delay) + } else { + log.Printf("[INFO] Fallback for model %s still shows %v instead of %v, retrying in %v", id, got, want, delay) + } + // the read cleared the ID and may have stored the old models; put back what was written + d.SetId(id) + if err := d.Set("fallback_models", want); err != nil { + return err + } + time.Sleep(delay) + } + + d.SetId(id) + return fmt.Errorf("fallback for model %s was written but not read back with fallback_models %v after %d attempts; the proxy may still be reloading it, so re-running apply should resolve this", id, want, len(fallbackReadBackDelays)+1) } func resourceLiteLLMFallbackCreate(d *schema.ResourceData, m interface{}) error { diff --git a/terraform/provider/litellm/resource_fallback_test.go b/terraform/provider/litellm/resource_fallback_test.go index e2c90d25424..0bf2f6d89b0 100644 --- a/terraform/provider/litellm/resource_fallback_test.go +++ b/terraform/provider/litellm/resource_fallback_test.go @@ -5,7 +5,10 @@ import ( "net/http" "net/http/httptest" "reflect" + "strings" + "sync/atomic" "testing" + "time" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" ) @@ -178,3 +181,86 @@ func TestResourceLiteLLMFallbackDelete(t *testing.T) { t.Fatalf("expected ID cleared after delete, got %q", d.Id()) } } + +func withFastFallbackReadBack(t *testing.T) { + t.Helper() + saved := fallbackReadBackDelays + fallbackReadBackDelays = []time.Duration{time.Millisecond, time.Millisecond, time.Millisecond} + t.Cleanup(func() { fallbackReadBackDelays = saved }) +} + +// fallbackReplicaServer accepts the POST and answers the first `stale` reads with staleGet, the way a +// replica that has not reloaded the write answers, then with the written fallback. +func fallbackReplicaServer(t *testing.T, stale int32, staleGet func(http.ResponseWriter), written FallbackGetResponse) (*httptest.Server, *int32) { + t.Helper() + var reads int32 + mux := http.NewServeMux() + mux.HandleFunc("/fallback", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"message":"ok"}`)) + }) + mux.HandleFunc("/fallback/gpt-4", func(w http.ResponseWriter, r *http.Request) { + if atomic.AddInt32(&reads, 1) <= stale { + staleGet(w) + return + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(written) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return srv, &reads +} + +func TestResourceLiteLLMFallbackCreate_RetriesReadUntilReplicaHasIt(t *testing.T) { + withFastFallbackReadBack(t) + notFound := func(w http.ResponseWriter) { w.WriteHeader(http.StatusNotFound) } + srv, reads := fallbackReplicaServer(t, 2, notFound, FallbackGetResponse{Model: "gpt-4", FallbackModels: []string{"claude-3"}, FallbackType: "general"}) + + d := newFallbackTestResourceData(t, "gpt-4", []interface{}{"claude-3"}, "general") + if err := resourceLiteLLMFallbackCreate(d, NewClient(srv.URL, "test-key", true)); err != nil { + t.Fatalf("expected nil error, got: %v", err) + } + if d.Id() != "gpt-4" { + t.Fatalf("expected ID 'gpt-4', got %q", d.Id()) + } + if got := atomic.LoadInt32(reads); got != 3 { + t.Fatalf("expected 3 reads (2 stale, 1 fresh), got %d", got) + } +} + +func TestResourceLiteLLMFallbackUpdate_RetriesReadWhileOldModelsAreReturned(t *testing.T) { + withFastFallbackReadBack(t) + oldModels := func(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(FallbackGetResponse{Model: "gpt-4", FallbackModels: []string{"claude-3"}, FallbackType: "general"}) + } + srv, reads := fallbackReplicaServer(t, 1, oldModels, FallbackGetResponse{Model: "gpt-4", FallbackModels: []string{"claude-3", "gpt-3.5-turbo"}, FallbackType: "general"}) + + d := newFallbackTestResourceData(t, "gpt-4", []interface{}{"claude-3", "gpt-3.5-turbo"}, "general") + d.SetId("gpt-4") + if err := resourceLiteLLMFallbackUpdate(d, NewClient(srv.URL, "test-key", true)); err != nil { + t.Fatalf("expected nil error, got: %v", err) + } + if got := expandStringList(d.Get("fallback_models").([]interface{})); !reflect.DeepEqual(got, []string{"claude-3", "gpt-3.5-turbo"}) { + t.Fatalf("unexpected fallback_models %v", got) + } + if got := atomic.LoadInt32(reads); got != 2 { + t.Fatalf("expected 2 reads (1 stale, 1 fresh), got %d", got) + } +} + +func TestResourceLiteLLMFallbackCreate_GivesUpWhenNeverReadBack(t *testing.T) { + withFastFallbackReadBack(t) + notFound := func(w http.ResponseWriter) { w.WriteHeader(http.StatusNotFound) } + srv, reads := fallbackReplicaServer(t, 100, notFound, FallbackGetResponse{}) + + d := newFallbackTestResourceData(t, "gpt-4", []interface{}{"claude-3"}, "general") + err := resourceLiteLLMFallbackCreate(d, NewClient(srv.URL, "test-key", true)) + if err == nil || !strings.Contains(err.Error(), "not read back") { + t.Fatalf("expected a not-read-back error, got: %v", err) + } + if got, want := atomic.LoadInt32(reads), int32(len(fallbackReadBackDelays)+1); got != want { + t.Fatalf("expected %d reads, got %d", want, got) + } +}