From e0515e4bb92ac93480b5a0e0ffdb4d8235d970f7 Mon Sep 17 00:00:00 2001 From: Maximilian Rink Date: Fri, 9 Oct 2026 23:57:11 +0200 Subject: [PATCH 01/11] ci: add manual GHCR publish workflow --- .github/workflows/publish-homelab-otel.yml | 47 ++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/publish-homelab-otel.yml diff --git a/.github/workflows/publish-homelab-otel.yml b/.github/workflows/publish-homelab-otel.yml new file mode 100644 index 00000000000..a33de9ac060 --- /dev/null +++ b/.github/workflows/publish-homelab-otel.yml @@ -0,0 +1,47 @@ +name: Publish homelab OTEL compatibility image +on: + workflow_dispatch: + inputs: + publish: + description: Publish the digest-pinned image to GHCR + required: true + default: 'true' + type: boolean +permissions: + contents: read + packages: write +jobs: + publish: + if: ${{ inputs.publish == true }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Write compatibility Dockerfile + run: | + cat > homelab-otel.Dockerfile <<'EOF_DOCKERFILE' + FROM ghcr.io/berriai/litellm@sha256:3d88bd757134a00e8c00237b85a021ab1f6d54abb9c1df750f411dbe7256a500 + LABEL org.opencontainers.image.source="https://github.com/BerriAI/litellm/pull/45612" \ + org.opencontainers.image.revision="faa168d979b23328a395104fb8670d502b01299c" \ + org.opencontainers.image.base.name="ghcr.io/berriai/litellm" \ + org.opencontainers.image.base.digest="sha256:3d88bd757134a00e8c00237b85a021ab1f6d54abb9c1df750f411dbe7256a500" \ + org.opencontainers.image.licenses="MIT" \ + com.maxrink.patch.opentelemetry.sha256="b139f02779ada57ad72e798ccfd3748276956dd88549fbb83ba63261b6f3ce32" + COPY litellm/integrations/opentelemetry.py /app/.venv/lib/python3.13/site-packages/litellm/integrations/opentelemetry.py + EOF_DOCKERFILE + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: docker/build-push-action@v6 + with: + context: . + file: homelab-otel.Dockerfile + push: true + tags: ghcr.io/maxrink/litellm-otel-fix:faa168-pinned-1.104.2 + labels: | + org.opencontainers.image.source=https://github.com/BerriAI/litellm/pull/45612 + org.opencontainers.image.revision=faa168d979b23328a395104fb8670d502b01299c + org.opencontainers.image.base.digest=sha256:3d88bd757134a00e8c00237b85a021ab1f6d54abb9c1df750f411dbe7256a500 + org.opencontainers.image.licenses=MIT + com.maxrink.patch.opentelemetry.sha256=b139f02779ada57ad72e798ccfd3748276956dd88549fbb83ba63261b6f3ce32 From 143345c979a14470560152f5636f347ea2dfb19b Mon Sep 17 00:00:00 2001 From: Maximilian Rink Date: Sat, 10 Oct 2026 00:18:04 +0200 Subject: [PATCH 02/11] ci: remove temporary GHCR publisher from upstream patch branch --- .github/workflows/publish-homelab-otel.yml | 47 ---------------------- 1 file changed, 47 deletions(-) delete mode 100644 .github/workflows/publish-homelab-otel.yml diff --git a/.github/workflows/publish-homelab-otel.yml b/.github/workflows/publish-homelab-otel.yml deleted file mode 100644 index a33de9ac060..00000000000 --- a/.github/workflows/publish-homelab-otel.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Publish homelab OTEL compatibility image -on: - workflow_dispatch: - inputs: - publish: - description: Publish the digest-pinned image to GHCR - required: true - default: 'true' - type: boolean -permissions: - contents: read - packages: write -jobs: - publish: - if: ${{ inputs.publish == true }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Write compatibility Dockerfile - run: | - cat > homelab-otel.Dockerfile <<'EOF_DOCKERFILE' - FROM ghcr.io/berriai/litellm@sha256:3d88bd757134a00e8c00237b85a021ab1f6d54abb9c1df750f411dbe7256a500 - LABEL org.opencontainers.image.source="https://github.com/BerriAI/litellm/pull/45612" \ - org.opencontainers.image.revision="faa168d979b23328a395104fb8670d502b01299c" \ - org.opencontainers.image.base.name="ghcr.io/berriai/litellm" \ - org.opencontainers.image.base.digest="sha256:3d88bd757134a00e8c00237b85a021ab1f6d54abb9c1df750f411dbe7256a500" \ - org.opencontainers.image.licenses="MIT" \ - com.maxrink.patch.opentelemetry.sha256="b139f02779ada57ad72e798ccfd3748276956dd88549fbb83ba63261b6f3ce32" - COPY litellm/integrations/opentelemetry.py /app/.venv/lib/python3.13/site-packages/litellm/integrations/opentelemetry.py - EOF_DOCKERFILE - - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/build-push-action@v6 - with: - context: . - file: homelab-otel.Dockerfile - push: true - tags: ghcr.io/maxrink/litellm-otel-fix:faa168-pinned-1.104.2 - labels: | - org.opencontainers.image.source=https://github.com/BerriAI/litellm/pull/45612 - org.opencontainers.image.revision=faa168d979b23328a395104fb8670d502b01299c - org.opencontainers.image.base.digest=sha256:3d88bd757134a00e8c00237b85a021ab1f6d54abb9c1df750f411dbe7256a500 - org.opencontainers.image.licenses=MIT - com.maxrink.patch.opentelemetry.sha256=b139f02779ada57ad72e798ccfd3748276956dd88549fbb83ba63261b6f3ce32 From 063021ba402be1e90338106bccce5b9eb8862dbb Mon Sep 17 00:00:00 2001 From: Maximilian Rink Date: Sat, 10 Oct 2026 00:46:51 +0200 Subject: [PATCH 03/11] ci: register minimal homelab image publisher --- .../publish-homelab-otel-minimal.yml | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/publish-homelab-otel-minimal.yml diff --git a/.github/workflows/publish-homelab-otel-minimal.yml b/.github/workflows/publish-homelab-otel-minimal.yml new file mode 100644 index 00000000000..1c40c27ee8d --- /dev/null +++ b/.github/workflows/publish-homelab-otel-minimal.yml @@ -0,0 +1,22 @@ +name: Publish homelab minimal OTEL image +on: + workflow_dispatch: +permissions: + contents: read + packages: write +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: docker/build-push-action@v6 + with: + context: . + file: Dockerfile.homelab-minimal + push: true + tags: ghcr.io/${{ github.repository_owner }}/litellm-otel-fix-minimal:1.104.2 From 926c84e4e01a23da8c3ee6e5a10f6443459245fb Mon Sep 17 00:00:00 2001 From: Maximilian Rink Date: Sat, 10 Oct 2026 01:25:35 +0200 Subject: [PATCH 04/11] fix: skip attributes on ended otel spans --- litellm/integrations/opentelemetry.py | 3 +++ tests/unit/integrations/test_opentelemetry.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 90383ddb60d..870aa001a6b 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -2764,6 +2764,9 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): """ Safely sets an attribute on the span, ensuring the value is a primitive type. """ + is_recording = getattr(span, "is_recording", None) + if callable(is_recording) and not is_recording(): + return primitive_value: Final = self._cast_as_primitive_value_type(value) span.set_attribute(key, primitive_value) diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index 56049c53736..13cab0c6293 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -388,6 +388,22 @@ def test_v1_malformed_routing_metadata_preserves_inference_attributes(decision: assert not any(key.startswith("litellm.routing.") for key in finished.attributes) +def test_safe_set_attribute_skips_ended_and_non_recording_spans() -> None: + otel: Final = OpenTelemetry.__new__(OpenTelemetry) + recording_span: Final = MagicMock() + recording_span.is_recording.return_value = True + ended_span: Final = MagicMock() + ended_span.is_recording.return_value = False + non_recording_span: Final = trace.NonRecordingSpan(trace.INVALID_SPAN_CONTEXT) + + otel.safe_set_attribute(recording_span, "litellm.test", "recorded") + otel.safe_set_attribute(ended_span, "litellm.test", "skipped") + otel.safe_set_attribute(non_recording_span, "litellm.test", "skipped") + + recording_span.set_attribute.assert_called_once_with("litellm.test", "recorded") + ended_span.set_attribute.assert_not_called() + + class TestOpenTelemetryCostBreakdown(unittest.TestCase): def test_cost_breakdown_emitted_to_otel_span(self): """ From 00b5d2283c8f64d31528e02aecafbe1c74d84e30 Mon Sep 17 00:00:00 2001 From: Maximilian Rink Date: Sat, 10 Oct 2026 01:26:39 +0200 Subject: [PATCH 05/11] chore: keep homelab publisher out of upstream change --- .../publish-homelab-otel-minimal.yml | 22 ------------------- 1 file changed, 22 deletions(-) delete mode 100644 .github/workflows/publish-homelab-otel-minimal.yml diff --git a/.github/workflows/publish-homelab-otel-minimal.yml b/.github/workflows/publish-homelab-otel-minimal.yml deleted file mode 100644 index 1c40c27ee8d..00000000000 --- a/.github/workflows/publish-homelab-otel-minimal.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Publish homelab minimal OTEL image -on: - workflow_dispatch: -permissions: - contents: read - packages: write -jobs: - publish: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/build-push-action@v6 - with: - context: . - file: Dockerfile.homelab-minimal - push: true - tags: ghcr.io/${{ github.repository_owner }}/litellm-otel-fix-minimal:1.104.2 From 1433e00bcd8011d9463d331e22d2212633532a85 Mon Sep 17 00:00:00 2001 From: Maximilian Rink Date: Sat, 10 Oct 2026 01:33:40 +0200 Subject: [PATCH 06/11] fix: satisfy otel guard type discipline --- litellm/integrations/opentelemetry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 870aa001a6b..8da5dce7184 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -2764,7 +2764,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): """ Safely sets an attribute on the span, ensuring the value is a primitive type. """ - is_recording = getattr(span, "is_recording", None) + is_recording: Final = getattr(span, "is_recording", None) if callable(is_recording) and not is_recording(): return primitive_value: Final = self._cast_as_primitive_value_type(value) From 161ce2ad895c9a6df2f265bd2ce7c3f737c92df9 Mon Sep 17 00:00:00 2001 From: Maximilian Rink Date: Sat, 10 Oct 2026 08:36:28 +0200 Subject: [PATCH 07/11] fix: avoid identity writes after server span end --- litellm/integrations/otel/logger.py | 3 ++- tests/unit/integrations/test_opentelemetry.py | 24 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 498ebe655da..1019cda191b 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -748,7 +748,8 @@ class OpenTelemetryV2(CustomLogger): # path's ``async_pre_call_hook``). Idempotent. set_request_root_span(server_span) for key, value in bag.items(): - server_span.set_attribute(key, value) + if is_recordable_span(server_span) and server_span.is_recording(): + server_span.set_attribute(key, value) except Exception: pass diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index 13cab0c6293..637923ccba7 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -46,6 +46,8 @@ from litellm.integrations.opentelemetry import ( RAW_REQUEST_SPAN_NAME, _normalize_team_metadata_keys, ) +from litellm.integrations.otel import logger as otel_logger +from litellm.integrations.otel.logger import OpenTelemetryV2 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy import proxy_server from litellm.proxy._types import SpanAttributes @@ -404,6 +406,28 @@ def test_safe_set_attribute_skips_ended_and_non_recording_spans() -> None: ended_span.set_attribute.assert_not_called() +@pytest.mark.parametrize("recording", [True, False]) +def test_seed_request_identity_checks_server_span_recording(recording: bool) -> None: + logger = OpenTelemetryV2() + server_span = MagicMock() + server_span.is_recording.return_value = recording + with ( + patch.object(otel_logger.RequestIdentity, "from_user_api_key_auth", return_value=MagicMock()), + patch.object(otel_logger, "promoted_baggage", return_value={"team.id": "qa-team"}), + patch.object(otel_logger, "request_root_span", return_value=server_span), + patch.object(otel_logger, "is_recordable_span", return_value=True), + patch.object(otel_logger, "set_request_root_span"), + patch.object(otel_logger, "set_request_baggage", return_value=MagicMock()), + patch.object(otel_logger, "attach"), + ): + logger.seed_request_identity({}, model="qa-model") + + if recording: + server_span.set_attribute.assert_called_once_with("team.id", "qa-team") + else: + server_span.set_attribute.assert_not_called() + + class TestOpenTelemetryCostBreakdown(unittest.TestCase): def test_cost_breakdown_emitted_to_otel_span(self): """ From 5ff5b8adc1834379db21fd0ae51c7eb71d7e8754 Mon Sep 17 00:00:00 2001 From: Maximilian Rink Date: Sat, 10 Oct 2026 08:43:23 +0200 Subject: [PATCH 08/11] test: document mock lifecycle branch coverage --- tests/unit/integrations/test_opentelemetry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index 637923ccba7..d308476fd0e 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -407,7 +407,7 @@ def test_safe_set_attribute_skips_ended_and_non_recording_spans() -> None: @pytest.mark.parametrize("recording", [True, False]) -def test_seed_request_identity_checks_server_span_recording(recording: bool) -> None: +def test_seed_request_identity_checks_server_span_recording(recording: bool) -> None: # test-quality-ok: the hosted proxy fixture covers end-to-end behavior; this isolates both lifecycle branches logger = OpenTelemetryV2() server_span = MagicMock() server_span.is_recording.return_value = recording From e6c7aa96b6100de9d949a7f69d7a37bc89a7bcc7 Mon Sep 17 00:00:00 2001 From: Maximilian Rink Date: Sat, 10 Oct 2026 15:05:39 +0200 Subject: [PATCH 09/11] test(otel): cover V2 identity with real SDK spans --- tests/unit/integrations/test_opentelemetry.py | 39 +++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index d308476fd0e..2941b9d68f1 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -11,7 +11,7 @@ import unittest import weakref from datetime import datetime, timedelta, timezone from pathlib import Path -from types import MappingProxyType +from types import MappingProxyType, SimpleNamespace from typing import Final from unittest.mock import MagicMock, patch @@ -48,6 +48,7 @@ from litellm.integrations.opentelemetry import ( ) from litellm.integrations.otel import logger as otel_logger from litellm.integrations.otel.logger import OpenTelemetryV2 +from litellm.integrations.otel.plumbing.context import set_request_root_span from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy import proxy_server from litellm.proxy._types import SpanAttributes @@ -406,26 +407,24 @@ def test_safe_set_attribute_skips_ended_and_non_recording_spans() -> None: ended_span.set_attribute.assert_not_called() -@pytest.mark.parametrize("recording", [True, False]) -def test_seed_request_identity_checks_server_span_recording(recording: bool) -> None: # test-quality-ok: the hosted proxy fixture covers end-to-end behavior; this isolates both lifecycle branches - logger = OpenTelemetryV2() - server_span = MagicMock() - server_span.is_recording.return_value = recording - with ( - patch.object(otel_logger.RequestIdentity, "from_user_api_key_auth", return_value=MagicMock()), - patch.object(otel_logger, "promoted_baggage", return_value={"team.id": "qa-team"}), - patch.object(otel_logger, "request_root_span", return_value=server_span), - patch.object(otel_logger, "is_recordable_span", return_value=True), - patch.object(otel_logger, "set_request_root_span"), - patch.object(otel_logger, "set_request_baggage", return_value=MagicMock()), - patch.object(otel_logger, "attach"), - ): - logger.seed_request_identity({}, model="qa-model") +def test_seed_request_identity_preserves_recording_span_and_ignores_ended_span() -> None: + exporter: Final = InMemorySpanExporter() + provider: Final = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + logger: Final = OpenTelemetryV2(tracer_provider=provider) + auth = SimpleNamespace(team_id="qa-team", api_key="qa-key") - if recording: - server_span.set_attribute.assert_called_once_with("team.id", "qa-team") - else: - server_span.set_attribute.assert_not_called() + with provider.get_tracer(__name__).start_as_current_span("server") as server_span: + set_request_root_span(server_span) + logger.seed_request_identity(auth, model="qa-model") + assert server_span.is_recording() + assert server_span.attributes["litellm.team.id"] == "qa-team" + + (finished,) = exporter.get_finished_spans() + attributes_before = dict(finished.attributes) + assert not server_span.is_recording() + logger.seed_request_identity(auth, model="qa-model") + assert dict(finished.attributes) == attributes_before class TestOpenTelemetryCostBreakdown(unittest.TestCase): From 8e119e64cffe2131ebe704a8444cf8265bf14b80 Mon Sep 17 00:00:00 2001 From: Maximilian Rink Date: Sun, 11 Oct 2026 04:18:44 +0200 Subject: [PATCH 10/11] test otel ended spans with SDK spans --- tests/unit/integrations/test_opentelemetry.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index 2941b9d68f1..0edc1525d5d 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -393,18 +393,21 @@ def test_v1_malformed_routing_metadata_preserves_inference_attributes(decision: def test_safe_set_attribute_skips_ended_and_non_recording_spans() -> None: otel: Final = OpenTelemetry.__new__(OpenTelemetry) - recording_span: Final = MagicMock() - recording_span.is_recording.return_value = True - ended_span: Final = MagicMock() - ended_span.is_recording.return_value = False + provider: Final = TracerProvider() + tracer: Final = provider.get_tracer(__name__) + + with tracer.start_as_current_span("recording") as recording_span: + otel.safe_set_attribute(recording_span, "litellm.test", "recorded") + assert recording_span.attributes["litellm.test"] == "recorded" + + ended_attributes: Final = dict(recording_span.attributes) + assert not recording_span.is_recording() non_recording_span: Final = trace.NonRecordingSpan(trace.INVALID_SPAN_CONTEXT) - otel.safe_set_attribute(recording_span, "litellm.test", "recorded") - otel.safe_set_attribute(ended_span, "litellm.test", "skipped") + otel.safe_set_attribute(recording_span, "litellm.test", "skipped") otel.safe_set_attribute(non_recording_span, "litellm.test", "skipped") - recording_span.set_attribute.assert_called_once_with("litellm.test", "recorded") - ended_span.set_attribute.assert_not_called() + assert dict(recording_span.attributes) == ended_attributes def test_seed_request_identity_preserves_recording_span_and_ignores_ended_span() -> None: From db8ab02ba94da2114f98acd0bdc9d088362cd3c8 Mon Sep 17 00:00:00 2001 From: Maximilian Rink Date: Tue, 13 Oct 2026 21:04:43 +0200 Subject: [PATCH 11/11] test(otel): use real ended SDK spans --- tests/unit/integrations/test_opentelemetry.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index a8e610dea42..87438893dff 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -391,9 +391,9 @@ def test_v1_malformed_routing_metadata_preserves_inference_attributes(decision: def test_safe_set_attribute_skips_ended_and_non_recording_spans() -> None: - otel: Final = OpenTelemetry.__new__(OpenTelemetry) provider: Final = TracerProvider() tracer: Final = provider.get_tracer(__name__) + otel: Final = OpenTelemetry(tracer_provider=provider) with tracer.start_as_current_span("recording") as recording_span: otel.safe_set_attribute(recording_span, "litellm.test", "recorded") @@ -401,7 +401,9 @@ def test_safe_set_attribute_skips_ended_and_non_recording_spans() -> None: ended_attributes: Final = dict(recording_span.attributes) assert not recording_span.is_recording() - non_recording_span: Final = trace.NonRecordingSpan(trace.INVALID_SPAN_CONTEXT) + with tracer.start_as_current_span("non-recording") as non_recording_span: + pass + assert not non_recording_span.is_recording() otel.safe_set_attribute(recording_span, "litellm.test", "skipped") otel.safe_set_attribute(non_recording_span, "litellm.test", "skipped")