mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
fix(otel-v2): emit GenAI message-content events for event_only / span_and_event
The V2 OpenTelemetry integration honored only the span half of
OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT. The mode drove
capture_span_content (which stamps gen_ai.input.messages /
gen_ai.output.messages), but nothing consumed the event half of the OTel
ContentCapturingMode contract; enable_events was dead and the engine emitted no
GenAI events. So span_and_event produced span attributes but no events, and
event_only produced neither, dropping the prompt and completion entirely.
Wire the capture mode to event emission: under event_only and span_and_event the
engine now records one gen_ai.{role}.message event per request message and one
gen_ai.choice per response choice on the LLM-call span, each carrying the
provider, role, and textual content. They are span events so they reach whatever
trace exporter is already configured, including the console exporter used to
reproduce the issue.
To keep event_only from leaking content onto span attributes, the retained
bodies are split from their span destination: the payload parser keeps the raw
prompt/response whenever either sink is active (capture_content), and a new
content_on_span flag gates whether the mappers may stamp them as attributes, so
all five content vocabularies stay correct across every mode. no_content and
span_only behavior is unchanged.
Fixes #30956
This commit is contained in:
parent
6437b812be
commit
a982517eb7
14 changed files with 325 additions and 17 deletions
|
|
@ -10,6 +10,7 @@ from opentelemetry.trace.status import Status, StatusCode
|
|||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
|
||||
from litellm.integrations.otel.mappers import resolve_mappers
|
||||
from litellm.integrations.otel.mappers.base import AttributeMapper, SpanData
|
||||
from litellm.integrations.otel.model.events import llm_message_events
|
||||
from litellm.integrations.otel.model.payloads import (
|
||||
GuardrailSpanData,
|
||||
LLMCallSpanData,
|
||||
|
|
@ -165,6 +166,14 @@ class SpanEmitter:
|
|||
for mapper in self._mappers:
|
||||
for key, value in mapper.map(data).items():
|
||||
span.set_attribute(key, value)
|
||||
# The event half of ``capture_message_content`` (``event_only`` /
|
||||
# ``span_and_event``): record prompt/response bodies as span events so
|
||||
# they reach the configured trace exporter even when content is kept off
|
||||
# the span attributes (``event_only``). Span-attribute content is the
|
||||
# mappers' job above and is gated separately by ``content_on_span``.
|
||||
if self._config.capture_event_content and isinstance(data, LLMCallSpanData):
|
||||
for name, attributes in llm_message_events(data):
|
||||
span.add_event(name, attributes)
|
||||
error = (
|
||||
data.error
|
||||
if isinstance(
|
||||
|
|
|
|||
|
|
@ -321,7 +321,9 @@ class OpenTelemetryV2(CustomLogger):
|
|||
carrier.span.end(end_time=to_ns(end_time))
|
||||
return None
|
||||
data = LLMCallSpanData.from_standard_logging_payload(
|
||||
payload, capture_content=self.config.capture_span_content
|
||||
payload,
|
||||
capture_content=self.config.capture_content,
|
||||
content_on_span=self.config.capture_span_content,
|
||||
)
|
||||
end_time_ns = to_ns(end_time)
|
||||
if carrier.span is not None:
|
||||
|
|
|
|||
|
|
@ -52,8 +52,10 @@ class GenAIMapper:
|
|||
else None
|
||||
),
|
||||
GenAI.REQUEST_SEED: lambda d: d.request_params.seed,
|
||||
GenAI.INPUT_MESSAGES: lambda d: serialize_messages(d.messages_in),
|
||||
GenAI.OUTPUT_MESSAGES: lambda d: serialize_messages(output_messages(d)),
|
||||
GenAI.INPUT_MESSAGES: lambda d: serialize_messages(d.span_messages_in),
|
||||
GenAI.OUTPUT_MESSAGES: lambda d: serialize_messages(
|
||||
output_messages(d.span_choices_out)
|
||||
),
|
||||
GenAI.RESPONSE_MODEL: lambda d: d.response_model,
|
||||
GenAI.RESPONSE_ID: lambda d: d.response_id,
|
||||
GenAI.RESPONSE_FINISH_REASONS: lambda d: (
|
||||
|
|
|
|||
|
|
@ -57,8 +57,10 @@ class LangfuseMapper:
|
|||
"langfuse.observation.model.parameters": lambda d: json_if(
|
||||
collect(LangfuseMapper._MODEL_PARAMS, d.request_params)
|
||||
),
|
||||
"langfuse.observation.input": lambda d: serialize_messages(d.messages_in),
|
||||
"langfuse.observation.output": lambda d: serialize_messages(output_messages(d)),
|
||||
"langfuse.observation.input": lambda d: serialize_messages(d.span_messages_in),
|
||||
"langfuse.observation.output": lambda d: serialize_messages(
|
||||
output_messages(d.span_choices_out)
|
||||
),
|
||||
"langfuse.observation.usage_details": lambda d: json_if(
|
||||
collect(LangfuseMapper._USAGE_FIELDS, d.usage)
|
||||
),
|
||||
|
|
|
|||
|
|
@ -42,10 +42,12 @@ class LangtraceMapper:
|
|||
|
||||
_BLOB_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
|
||||
"llm.prompts": lambda d: (
|
||||
json_or_none(list(d.messages_in)) if d.messages_in else None
|
||||
json_or_none(list(d.span_messages_in)) if d.span_messages_in else None
|
||||
),
|
||||
"llm.completions": lambda d: (
|
||||
json_or_none(output_messages(d)) if d.choices_out else None
|
||||
json_or_none(output_messages(d.span_choices_out))
|
||||
if d.span_choices_out
|
||||
else None
|
||||
),
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -82,9 +82,11 @@ class OpenInferenceMapper:
|
|||
return {
|
||||
**collect(cls._LLM_CALL_ATTRS, data),
|
||||
**collect(cls._BLOB_ATTRS, data),
|
||||
**cls._messages("llm.input_messages", "input.value", data.messages_in),
|
||||
**cls._messages("llm.input_messages", "input.value", data.span_messages_in),
|
||||
**cls._messages(
|
||||
"llm.output_messages", "output.value", output_messages(data)
|
||||
"llm.output_messages",
|
||||
"output.value",
|
||||
output_messages(data.span_choices_out),
|
||||
),
|
||||
**cls._tools(data),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ import json
|
|||
from typing import Callable, Mapping, Sequence
|
||||
|
||||
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue
|
||||
from litellm.integrations.otel.model.payloads import LLMCallSpanData
|
||||
|
||||
|
||||
def drop_none(values: Mapping[str, AttrValue | None]) -> AttributeMap:
|
||||
|
|
@ -71,6 +70,6 @@ def message_content(message: object) -> str | None:
|
|||
return None
|
||||
|
||||
|
||||
def output_messages(data: LLMCallSpanData) -> list:
|
||||
def output_messages(choices: Sequence[Mapping[str, object]]) -> list[object]:
|
||||
"""The ``message`` payload of each response choice."""
|
||||
return [c.get("message") for c in data.choices_out if isinstance(c, dict)]
|
||||
return [c.get("message") for c in choices]
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ class WeaveMapper:
|
|||
_BLOB_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
|
||||
# Weave treats the response choices as the "output" payload.
|
||||
"weave.output": lambda d: (
|
||||
json_or_none(list(d.choices_out)) if d.choices_out else None
|
||||
json_or_none(list(d.span_choices_out)) if d.span_choices_out else None
|
||||
),
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -288,6 +288,30 @@ class OpenTelemetryV2Config(BaseSettings):
|
|||
CaptureMessageContent.SPAN_AND_EVENT,
|
||||
)
|
||||
|
||||
@property
|
||||
def capture_event_content(self) -> bool:
|
||||
"""Whether prompt/response content may be emitted as span events.
|
||||
|
||||
The event half of the OTel ``ContentCapturingMode`` contract: under
|
||||
``event_only`` and ``span_and_event`` the message bodies are recorded as
|
||||
``gen_ai.*`` events on the LLM-call span, complementing (or, for
|
||||
``event_only``, standing in for) the ``gen_ai.input.messages`` /
|
||||
``gen_ai.output.messages`` span attributes that ``capture_span_content``
|
||||
governs. Defaults off, like its span counterpart.
|
||||
"""
|
||||
return self.capture_message_content in (
|
||||
CaptureMessageContent.EVENT_ONLY,
|
||||
CaptureMessageContent.SPAN_AND_EVENT,
|
||||
)
|
||||
|
||||
@property
|
||||
def capture_content(self) -> bool:
|
||||
"""Whether message bodies are retained at all — as span attributes, span
|
||||
events, or both. The payload parser keeps the raw prompt/response only
|
||||
when this is true, so a request can never force content past either sink
|
||||
while every capture mode is off."""
|
||||
return self.capture_span_content or self.capture_event_content
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "OpenTelemetryV2Config":
|
||||
return cls()
|
||||
|
|
|
|||
73
litellm/integrations/otel/model/events.py
Normal file
73
litellm/integrations/otel/model/events.py
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
"""GenAI message events: the event half of ``capture_message_content``.
|
||||
|
||||
The OTel GenAI ``ContentCapturingMode`` records prompt/response content either as
|
||||
span attributes (``gen_ai.input.messages`` / ``gen_ai.output.messages``, owned by
|
||||
the mappers) or as events, or both. This module owns the event representation:
|
||||
one ``gen_ai.{role}.message`` event per request message and one ``gen_ai.choice``
|
||||
event per response choice, stamped on the LLM-call span so they ride whatever
|
||||
trace exporter the operator already configured — the console exporter included —
|
||||
without a separate logs pipeline.
|
||||
"""
|
||||
|
||||
from typing import Mapping, cast
|
||||
|
||||
from litellm.integrations.otel.mappers.base import AttributeMap
|
||||
from litellm.integrations.otel.mappers.utils import drop_none, message_content
|
||||
from litellm.integrations.otel.model.payloads import LLMCallSpanData
|
||||
from litellm.integrations.otel.model.semconv import GenAI, GenAIEvent
|
||||
from litellm.integrations.otel.model.utils import as_str
|
||||
|
||||
|
||||
def _role(message: object) -> str | None:
|
||||
"""The chat-message ``role``, or ``None`` when absent or not a dict."""
|
||||
if not isinstance(message, dict):
|
||||
return None
|
||||
role = cast(Mapping[str, object], message).get("role")
|
||||
return role if isinstance(role, str) and role else None
|
||||
|
||||
|
||||
def llm_message_events(
|
||||
data: LLMCallSpanData,
|
||||
) -> tuple[tuple[str, AttributeMap], ...]:
|
||||
"""``(event_name, attributes)`` pairs for an LLM call's prompt and response.
|
||||
|
||||
One ``gen_ai.{role}.message`` event per request message and one
|
||||
``gen_ai.choice`` per response choice, each carrying the provider, role, and
|
||||
(when present) the textual content. The caller stamps them as span events.
|
||||
"""
|
||||
provider = data.provider or None
|
||||
inputs = tuple(_input_event(provider, m) for m in data.messages_in)
|
||||
outputs = tuple(
|
||||
_choice_event(provider, idx, c) for idx, c in enumerate(data.choices_out)
|
||||
)
|
||||
return inputs + outputs
|
||||
|
||||
|
||||
def _input_event(
|
||||
provider: str | None, message: Mapping[str, object]
|
||||
) -> tuple[str, AttributeMap]:
|
||||
role = _role(message) or "user"
|
||||
attrs = drop_none(
|
||||
{
|
||||
GenAI.PROVIDER_NAME: provider,
|
||||
"role": role,
|
||||
"content": message_content(message),
|
||||
}
|
||||
)
|
||||
return GenAIEvent.message(role), attrs
|
||||
|
||||
|
||||
def _choice_event(
|
||||
provider: str | None, index: int, choice: Mapping[str, object]
|
||||
) -> tuple[str, AttributeMap]:
|
||||
message = choice.get("message")
|
||||
attrs = drop_none(
|
||||
{
|
||||
GenAI.PROVIDER_NAME: provider,
|
||||
"index": index,
|
||||
"finish_reason": as_str(choice.get("finish_reason")),
|
||||
"role": _role(message),
|
||||
"content": message_content(message),
|
||||
}
|
||||
)
|
||||
return GenAIEvent.CHOICE, attrs
|
||||
|
|
@ -308,11 +308,31 @@ class LLMCallSpanData:
|
|||
# dataclass stays hashable and frozen.
|
||||
messages_in: tuple[Mapping[str, object], ...] = ()
|
||||
choices_out: tuple[Mapping[str, object], ...] = ()
|
||||
# Whether the retained bodies may be stamped as span *attributes*. False
|
||||
# under ``event_only`` capture, where the bodies are kept for span events
|
||||
# only and must not leak onto ``gen_ai.input.messages`` / ``...output...``
|
||||
# (or any vendor vocabulary's) span attributes.
|
||||
content_on_span: bool = True
|
||||
system_fingerprint: str | None = None
|
||||
|
||||
@property
|
||||
def span_messages_in(self) -> tuple[Mapping[str, object], ...]:
|
||||
"""Request messages exposed to span-attribute mappers — empty when the
|
||||
captured content is bound to events only."""
|
||||
return self.messages_in if self.content_on_span else ()
|
||||
|
||||
@property
|
||||
def span_choices_out(self) -> tuple[Mapping[str, object], ...]:
|
||||
"""Response choices exposed to span-attribute mappers — empty when the
|
||||
captured content is bound to events only."""
|
||||
return self.choices_out if self.content_on_span else ()
|
||||
|
||||
@classmethod
|
||||
def from_standard_logging_payload(
|
||||
cls, payload: "StandardLoggingPayload", capture_content: bool = False
|
||||
cls,
|
||||
payload: "StandardLoggingPayload",
|
||||
capture_content: bool = False,
|
||||
content_on_span: bool | None = None,
|
||||
) -> "LLMCallSpanData":
|
||||
params = cast(Mapping[str, object], payload.get("model_parameters") or {})
|
||||
# The single parse of the request's metadata — the request-vs-provider
|
||||
|
|
@ -328,9 +348,12 @@ class LLMCallSpanData:
|
|||
choices_out = _dicts(response.get("choices"))
|
||||
# ``finish_reasons`` is metadata, not content, so derive it from
|
||||
# ``choices_out`` before gating. The raw message/choice bodies are only
|
||||
# retained when content capture is enabled (see ``capture_span_content``);
|
||||
# otherwise the content-bearing mappers receive empty sequences and emit
|
||||
# no prompt/response text.
|
||||
# retained when content capture is enabled — for span attributes, span
|
||||
# events, or both (see ``capture_content``). ``content_on_span`` then
|
||||
# decides whether the bodies reach span attributes; under ``event_only``
|
||||
# they are retained but kept off the span, for events alone. With capture
|
||||
# off the content-bearing mappers receive empty sequences and emit no
|
||||
# prompt/response text.
|
||||
finish_reasons = _finish_reasons(choices_out)
|
||||
return cls(
|
||||
operation=resolve_operation(as_str(payload.get("call_type"))),
|
||||
|
|
@ -356,6 +379,9 @@ class LLMCallSpanData:
|
|||
tools=_extract_tools(params),
|
||||
messages_in=_dicts(payload.get("messages")) if capture_content else (),
|
||||
choices_out=choices_out if capture_content else (),
|
||||
content_on_span=(
|
||||
capture_content if content_on_span is None else content_on_span
|
||||
),
|
||||
system_fingerprint=as_str(response.get("system_fingerprint")),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -92,6 +92,20 @@ class GenAI:
|
|||
PROMPT_NAME: Final = "gen_ai.prompt.name"
|
||||
|
||||
|
||||
class GenAIEvent:
|
||||
"""OTel GenAI event names. Prompt/response content is recorded as these
|
||||
events on the LLM-call span under the ``event_only`` / ``span_and_event``
|
||||
capture modes — the event half of ``ContentCapturingMode``."""
|
||||
|
||||
CHOICE: Final = "gen_ai.choice"
|
||||
|
||||
@staticmethod
|
||||
def message(role: str) -> str:
|
||||
"""Per-message event name, e.g. ``gen_ai.user.message``. The standard
|
||||
roles (system/user/assistant/tool) yield the canonical semconv names."""
|
||||
return f"gen_ai.{role}.message"
|
||||
|
||||
|
||||
class MCP:
|
||||
"""OTel GenAI MCP (Model Context Protocol) span-attribute keys.
|
||||
|
||||
|
|
|
|||
|
|
@ -268,6 +268,67 @@ def test_genai_mapper_omits_messages_when_content_not_captured():
|
|||
assert GenAI.OUTPUT_MESSAGES not in attrs
|
||||
|
||||
|
||||
def _captured_llm_call(content_on_span=True):
|
||||
return LLMCallSpanData(
|
||||
operation=GenAIOperation.CHAT,
|
||||
provider="openai",
|
||||
request_model="gpt-4o",
|
||||
response_model="gpt-4o-2024",
|
||||
response_id="resp_1",
|
||||
request_params=LLMRequestParams(),
|
||||
usage=LLMUsage(),
|
||||
finish_reasons=("stop",),
|
||||
error=None,
|
||||
response_cost=None,
|
||||
server=None,
|
||||
identity=RequestIdentity(call_id="c1"),
|
||||
messages_in=(
|
||||
{"role": "system", "content": "Be concise."},
|
||||
{"role": "user", "content": "What's the weather?"},
|
||||
),
|
||||
choices_out=(
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"message": {"role": "assistant", "content": "Sunny."},
|
||||
},
|
||||
),
|
||||
content_on_span=content_on_span,
|
||||
)
|
||||
|
||||
|
||||
def test_genai_mapper_withholds_span_content_when_event_only():
|
||||
"""``content_on_span=False`` (the ``event_only`` mode) keeps retained bodies
|
||||
off the span attributes even though they're present for the event path."""
|
||||
attrs = GenAIMapper().map(_captured_llm_call(content_on_span=False))
|
||||
assert GenAI.INPUT_MESSAGES not in attrs
|
||||
assert GenAI.OUTPUT_MESSAGES not in attrs
|
||||
|
||||
|
||||
def test_llm_message_events_carry_role_and_content():
|
||||
from litellm.integrations.otel.model.events import llm_message_events
|
||||
|
||||
events = llm_message_events(_captured_llm_call())
|
||||
names = [name for name, _ in events]
|
||||
assert names == ["gen_ai.system.message", "gen_ai.user.message", "gen_ai.choice"]
|
||||
|
||||
by_name = {name: attrs for name, attrs in events}
|
||||
assert by_name["gen_ai.system.message"]["role"] == "system"
|
||||
assert by_name["gen_ai.system.message"]["content"] == "Be concise."
|
||||
assert by_name["gen_ai.user.message"]["content"] == "What's the weather?"
|
||||
choice = by_name["gen_ai.choice"]
|
||||
assert choice["index"] == 0
|
||||
assert choice["finish_reason"] == "stop"
|
||||
assert choice["role"] == "assistant"
|
||||
assert choice["content"] == "Sunny."
|
||||
assert choice[GenAI.PROVIDER_NAME] == "openai"
|
||||
|
||||
|
||||
def test_llm_message_events_empty_without_bodies():
|
||||
from litellm.integrations.otel.model.events import llm_message_events
|
||||
|
||||
assert llm_message_events(_full_llm_call()) == ()
|
||||
|
||||
|
||||
def test_genai_mapper_cost_breakdown():
|
||||
from litellm.integrations.otel.model.semconv import LiteLLM
|
||||
|
||||
|
|
|
|||
|
|
@ -238,6 +238,98 @@ def test_idempotent_on_repeat_callback():
|
|||
assert len(exporter.get_finished_spans()) == 1
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Message-content capture modes (issue #30956)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _logger_for_mode(mode):
|
||||
cfg = OpenTelemetryV2Config(
|
||||
exporter="in_memory",
|
||||
legacy_compat=False,
|
||||
capture_message_content=mode,
|
||||
)
|
||||
exporter = InMemorySpanExporter()
|
||||
tracer_provider = providers.build_tracer_provider(cfg, exporter=exporter)
|
||||
return OpenTelemetryV2(config=cfg, tracer_provider=tracer_provider), exporter
|
||||
|
||||
|
||||
def _content_payload():
|
||||
return _payload(
|
||||
messages=[
|
||||
{"role": "system", "content": "Be concise."},
|
||||
{"role": "user", "content": "What's the capital of France?"},
|
||||
],
|
||||
response={
|
||||
"id": "resp_1",
|
||||
"model": "gpt-4o-2024",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"message": {"role": "assistant", "content": "Paris."},
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _emit_content_span(mode):
|
||||
logger, exporter = _logger_for_mode(mode)
|
||||
_emit_llm(logger, _kwargs(_content_payload()))
|
||||
(span,) = exporter.get_finished_spans()
|
||||
return span
|
||||
|
||||
|
||||
def test_span_and_event_emits_both_attributes_and_events():
|
||||
"""Regression for #30956: ``span_and_event`` must put message content on the
|
||||
span attributes AND emit semantic events. Before the fix, only the span
|
||||
attributes were produced and no events were emitted."""
|
||||
span = _emit_content_span("span_and_event")
|
||||
attrs = dict(span.attributes or {})
|
||||
|
||||
assert "What's the capital of France?" in attrs[GenAI.INPUT_MESSAGES]
|
||||
assert "Paris." in attrs[GenAI.OUTPUT_MESSAGES]
|
||||
|
||||
events = {e.name: dict(e.attributes or {}) for e in span.events}
|
||||
assert set(events) == {
|
||||
"gen_ai.system.message",
|
||||
"gen_ai.user.message",
|
||||
"gen_ai.choice",
|
||||
}
|
||||
assert events["gen_ai.user.message"]["content"] == "What's the capital of France?"
|
||||
assert events["gen_ai.choice"]["content"] == "Paris."
|
||||
assert events["gen_ai.choice"]["finish_reason"] == "stop"
|
||||
|
||||
|
||||
def test_event_only_emits_events_without_span_content():
|
||||
"""Regression for #30956: ``event_only`` must still capture content — as
|
||||
events — instead of dropping it. Before the fix this mode produced neither
|
||||
span attributes nor events, so the content was lost entirely."""
|
||||
span = _emit_content_span("event_only")
|
||||
attrs = dict(span.attributes or {})
|
||||
|
||||
assert GenAI.INPUT_MESSAGES not in attrs
|
||||
assert GenAI.OUTPUT_MESSAGES not in attrs
|
||||
|
||||
events = {e.name for e in span.events}
|
||||
assert events == {"gen_ai.system.message", "gen_ai.user.message", "gen_ai.choice"}
|
||||
|
||||
|
||||
def test_span_only_emits_no_message_events():
|
||||
span = _emit_content_span("span_only")
|
||||
attrs = dict(span.attributes or {})
|
||||
assert GenAI.INPUT_MESSAGES in attrs
|
||||
assert [e for e in span.events if e.name.startswith("gen_ai.")] == []
|
||||
|
||||
|
||||
def test_no_content_emits_neither_attributes_nor_events():
|
||||
span = _emit_content_span("no_content")
|
||||
attrs = dict(span.attributes or {})
|
||||
assert GenAI.INPUT_MESSAGES not in attrs
|
||||
assert GenAI.OUTPUT_MESSAGES not in attrs
|
||||
assert [e for e in span.events if e.name.startswith("gen_ai.")] == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# MCP tool-call spans
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue