diff --git a/helm/litellm-helm/templates/migrations-job.yaml b/helm/litellm-helm/templates/migrations-job.yaml index f8a660e23f8..5a873cbb965 100644 --- a/helm/litellm-helm/templates/migrations-job.yaml +++ b/helm/litellm-helm/templates/migrations-job.yaml @@ -119,4 +119,7 @@ spec: {{- end }} ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }} backoffLimit: {{ .Values.migrationJob.backoffLimit }} + {{- with .Values.migrationJob.activeDeadlineSeconds }} + activeDeadlineSeconds: {{ . }} + {{- end }} {{- end }} diff --git a/helm/litellm-helm/tests/migrations-job_tests.yaml b/helm/litellm-helm/tests/migrations-job_tests.yaml index cb962118a25..e327a3ec201 100644 --- a/helm/litellm-helm/tests/migrations-job_tests.yaml +++ b/helm/litellm-helm/tests/migrations-job_tests.yaml @@ -314,3 +314,31 @@ tests: operator: Equal value: litellm-e2e effect: NoSchedule + + - it: bounds the Job with a deadline by default, so a blocked migration cannot stall the release forever + set: + migrationJob: + enabled: true + asserts: + - equal: + path: spec.activeDeadlineSeconds + value: 1800 + + - it: honours an operator-supplied deadline + set: + migrationJob: + enabled: true + activeDeadlineSeconds: 600 + asserts: + - equal: + path: spec.activeDeadlineSeconds + value: 600 + + - it: omits the deadline entirely when it is nulled out, restoring the unbounded behaviour + set: + migrationJob: + enabled: true + activeDeadlineSeconds: null + asserts: + - notExists: + path: spec.activeDeadlineSeconds diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index df2b55723fe..628ca038339 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -427,6 +427,13 @@ migrationJob: enabled: true # Enable or disable the schema migration Job retries: 3 # Number of retries for the Job in case of failure backoffLimit: 4 # Backoff limit for Job restarts + # Wall-clock budget for the whole Job, shared across every `backoffLimit` + # retry rather than granted per attempt. Without it a migration that blocks + # on the database never fails, and when the Helm hook is enabled the release + # waits on it forever: `helm upgrade` and any GitOps controller driving it + # stop reconciling the whole chart until someone deletes the Job by hand. + # Set to null to opt out and restore the unbounded behaviour. + activeDeadlineSeconds: 1800 disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0. # Optional service account for the migration job. # Only used when migrationJob.hooks.helm.enabled=true and serviceAccount.create=true. diff --git a/helm/litellm/templates/migrations-job.yaml b/helm/litellm/templates/migrations-job.yaml index 2debe8a1e10..9cd8397f794 100644 --- a/helm/litellm/templates/migrations-job.yaml +++ b/helm/litellm/templates/migrations-job.yaml @@ -21,6 +21,9 @@ metadata: spec: backoffLimit: {{ .Values.migrationJob.backoffLimit }} ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }} + {{- with .Values.migrationJob.activeDeadlineSeconds }} + activeDeadlineSeconds: {{ . }} + {{- end }} template: metadata: {{- /* The Job's selector is generated by the controller rather than diff --git a/helm/litellm/tests/migration_job_tests.yaml b/helm/litellm/tests/migration_job_tests.yaml index 12e525c5a8c..c3f3083ece5 100644 --- a/helm/litellm/tests/migration_job_tests.yaml +++ b/helm/litellm/tests/migration_job_tests.yaml @@ -167,3 +167,24 @@ tests: - equal: path: spec.template.metadata.labels['app.kubernetes.io/component'] value: batch-migrations + + - it: bounds the Job with a deadline by default, so a blocked migration cannot stall the release forever + asserts: + - equal: + path: spec.activeDeadlineSeconds + value: 1800 + + - it: honours an operator-supplied deadline + set: + migrationJob.activeDeadlineSeconds: 600 + asserts: + - equal: + path: spec.activeDeadlineSeconds + value: 600 + + - it: omits the deadline entirely when it is nulled out, restoring the unbounded behaviour + set: + migrationJob.activeDeadlineSeconds: null + asserts: + - notExists: + path: spec.activeDeadlineSeconds diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 7820a898ef1..3f8aacfce17 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -56,6 +56,15 @@ migrationJob: enabled: true backoffLimit: 4 ttlSecondsAfterFinished: 120 + # Wall-clock budget for the whole Job, shared across every `backoffLimit` + # retry rather than granted per attempt. Without it a migration that blocks + # on the database never fails, and because this is a pre-upgrade hook the + # release waits on it forever: `helm upgrade` and any GitOps controller + # driving it stop reconciling the whole chart until someone deletes the Job + # by hand. A migration that has exhausted its retries is not going to + # succeed on the next one, so failing is strictly better than hanging. + # Set to null to opt out and restore the unbounded behaviour. + activeDeadlineSeconds: 1800 resources: {} # ServiceAccount for the Job pod only. #