ai-workspace-services/litellm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added user_email labels to the prometheus monitoring.

Browse Source
This commit is contained in:
boopesh07 2025-09-12 15:38:23 -07:00
parent 6a47ac15ab
commit 36299dbc73
3 changed files with 153 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
litellm/types/integrations/prometheus.py
View File

@ -240,6 +240,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.TEAM.value,
UserAPIKeyLabelNames.TEAM_ALIAS.value,
UserAPIKeyLabelNames.USER.value,
UserAPIKeyLabelNames.USER_EMAIL.value,
UserAPIKeyLabelNames.EXCEPTION_STATUS.value,
UserAPIKeyLabelNames.EXCEPTION_CLASS.value,
UserAPIKeyLabelNames.ROUTE.value,
@ -302,6 +303,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.TEAM.value,
UserAPIKeyLabelNames.TEAM_ALIAS.value,
UserAPIKeyLabelNames.USER.value,
UserAPIKeyLabelNames.USER_EMAIL.value,
]
litellm_input_tokens_metric = [
@ -312,6 +314,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.TEAM.value,
UserAPIKeyLabelNames.TEAM_ALIAS.value,
UserAPIKeyLabelNames.USER.value,
UserAPIKeyLabelNames.USER_EMAIL.value,
UserAPIKeyLabelNames.REQUESTED_MODEL.value,
]
@ -323,6 +326,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.TEAM.value,
UserAPIKeyLabelNames.TEAM_ALIAS.value,
UserAPIKeyLabelNames.USER.value,
UserAPIKeyLabelNames.USER_EMAIL.value,
UserAPIKeyLabelNames.REQUESTED_MODEL.value,
]
@ -334,6 +338,7 @@ class PrometheusMetricLabels:
UserAPIKeyLabelNames.TEAM.value,
UserAPIKeyLabelNames.TEAM_ALIAS.value,
UserAPIKeyLabelNames.USER.value,
UserAPIKeyLabelNames.USER_EMAIL.value,
UserAPIKeyLabelNames.REQUESTED_MODEL.value,
]

73
tests/otel_tests/test_prometheus.py
View File

@ -106,7 +106,7 @@ async def test_proxy_failure_metrics():
print("/metrics", metrics)
# Check if the failure metric is present and correct
expected_metric = 'litellm_proxy_failed_requests_metric_total{api_key_alias="None",end_user="None",exception_class="Openai.RateLimitError",exception_status="429",hashed_api_key="88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",requested_model="fake-azure-endpoint",route="/chat/completions",team="None",team_alias="None",user="default_user_id"} 1.0'
expected_metric = 'litellm_proxy_failed_requests_metric_total{api_key_alias="None",end_user="None",exception_class="Openai.RateLimitError",exception_status="429",hashed_api_key="88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",requested_model="fake-azure-endpoint",route="/chat/completions",team="None",team_alias="None",user="default_user_id", user_email="None"} 1.0'
assert (
expected_metric in metrics
@ -577,3 +577,74 @@ async def test_user_email_metrics():
assert (
user_email in metrics_after_first
), "user_email should be tracked correctly"
@pytest.mark.asyncio
async def test_user_email_in_all_required_metrics():
"""
Test that user_email label is present in all the metrics that were requested to have it:
- litellm_proxy_total_requests_metric_total
- litellm_proxy_failed_requests_metric_total
- litellm_input_tokens_total
- litellm_output_tokens_total
- litellm_requests_metric_total
- litellm_spend_metric_total
"""
async with aiohttp.ClientSession() as session:
# Create a user with user_email
user_email = f"test-metrics-{uuid.uuid4()}@example.com"
user_data = {
"user_email": user_email,
}
user_info = await create_test_user(session, user_data)
key = user_info["key"]
# Initialize OpenAI client with the user's email
client = AsyncOpenAI(base_url="http://0.0.0.0:4000", api_key=key)
# Make successful request to generate metrics
await client.chat.completions.create(
model="fake-openai-endpoint",
messages=[{"role": "user", "content": f"Hello {uuid.uuid4()}"}],
)
await asyncio.sleep(11) # Wait for metrics to update
# Get metrics after request
metrics_text = await get_prometheus_metrics(session)
print("Testing user_email in all required metrics")
# Check that user_email appears in all the required metrics
required_metrics_with_user_email = [
"litellm_proxy_total_requests_metric_total",
"litellm_input_tokens_total",
"litellm_output_tokens_total",
"litellm_requests_metric_total",
"litellm_spend_metric_total"
]
for metric_name in required_metrics_with_user_email:
# Check that the metric exists and contains user_email label
import re
# Look for the metric with user_email in its labels
pattern = rf'{metric_name}{{[^}}]*user_email="{re.escape(user_email)}"[^}}]*}}'
matches = re.findall(pattern, metrics_text)
assert len(matches) > 0, f"Metric {metric_name} should contain user_email={user_email} but was not found in metrics"
# Also test failure metric by making a bad request
try:
await client.chat.completions.create(
model="fake-azure-endpoint", # This should fail
messages=[{"role": "user", "content": "Hello"}],
)
except Exception:
pass # Expected to fail
await asyncio.sleep(11) # Wait for metrics to update
# Get updated metrics
metrics_text = await get_prometheus_metrics(session)
# Check that failure metric also contains user_email
failure_pattern = rf'litellm_proxy_failed_requests_metric_total{{[^}}]*user_email="{re.escape(user_email)}"[^}}]*}}'
failure_matches = re.findall(failure_pattern, metrics_text)
assert len(failure_matches) > 0, f"litellm_proxy_failed_requests_metric_total should contain user_email={user_email}"

76
tests/test_litellm/integrations/test_prometheus_labels.py Normal file
View File

@ -0,0 +1,76 @@
"""
Unit tests for prometheus metric labels configuration
"""
from litellm.types.integrations.prometheus import (
PrometheusMetricLabels,
UserAPIKeyLabelNames
)
def test_user_email_in_required_metrics():
"""
Test that user_email label is present in all the metrics that should have it:
- litellm_proxy_total_requests_metric (already had it)
- litellm_proxy_failed_requests_metric (added)
- litellm_input_tokens_metric (added)
- litellm_output_tokens_metric (added)
- litellm_requests_metric (already had it)
- litellm_spend_metric (added)
"""
user_email_label = UserAPIKeyLabelNames.USER_EMAIL.value
# Metrics that should have user_email
metrics_with_user_email = [
"litellm_proxy_total_requests_metric",
"litellm_proxy_failed_requests_metric",
"litellm_input_tokens_metric",
"litellm_output_tokens_metric",
"litellm_requests_metric",
"litellm_spend_metric"
]
for metric_name in metrics_with_user_email:
labels = PrometheusMetricLabels.get_labels(metric_name)
assert user_email_label in labels, f"Metric {metric_name} should contain user_email label"
print(f"✅ {metric_name} contains user_email label")
def test_user_email_label_exists():
"""Test that the USER_EMAIL label is properly defined"""
assert UserAPIKeyLabelNames.USER_EMAIL.value == "user_email"
def test_prometheus_metric_labels_structure():
"""Test that all required prometheus metrics have proper label structure"""
from litellm.types.integrations.prometheus import DEFINED_PROMETHEUS_METRICS
from typing import get_args
# Test a few key metrics to ensure they have proper label structure
test_metrics = [
"litellm_proxy_total_requests_metric",
"litellm_proxy_failed_requests_metric",
"litellm_input_tokens_metric",
"litellm_output_tokens_metric",
"litellm_spend_metric"
]
for metric_name in test_metrics:
# Check metric is in DEFINED_PROMETHEUS_METRICS
assert metric_name in get_args(DEFINED_PROMETHEUS_METRICS), f"{metric_name} should be in DEFINED_PROMETHEUS_METRICS"
# Check labels can be retrieved
labels = PrometheusMetricLabels.get_labels(metric_name)
assert isinstance(labels, list), f"Labels for {metric_name} should be a list"
assert len(labels) > 0, f"Labels for {metric_name} should not be empty"
# Check user_email is in the labels
assert "user_email" in labels, f"{metric_name} should have user_email label"
print(f"✅ {metric_name} has proper label structure with user_email")
if __name__ == "__main__":
test_user_email_in_required_metrics()
test_user_email_label_exists()
test_prometheus_metric_labels_structure()
print("All prometheus label tests passed!")