ai-workspace-services/litellm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Request Headers - support x-litellm-num-retries + Usage - support usage by model group (#12890)

Browse Source 
* feat(litellm_pre_call_utils.py): add num_retries to litellm data for backend call

allow user to pass in num retries via request headers

* test(test_litellm_pre_call_utils.py): add unit test

* docs(request_headers.md): document new request header

* fix(common_daily_activity.py): show spend breakdown by model group

Partial fix for https://github.com/BerriAI/litellm/issues/12887

* feat(new_usage.tsx): new tab switcher for viewing usage by model group vs. received model

Closes https://github.com/BerriAI/litellm/issues/12887
This commit is contained in:
Krish Dholakia 2025-07-22 23:09:36 -07:00 committed by GitHub
parent a0846c3f31
commit d26f74ade5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 315 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/my-website/docs/proxy/request_headers.md
View File

@ -10,6 +10,8 @@ Special headers that are supported by LiteLLM.
`x-litellm-tags`: Optional[str]: A comma separated list (e.g. `tag1,tag2,tag3`) of tags to use for [tag-based routing](./tag_routing) **OR** [spend-tracking](./enterprise.md#tracking-spend-for-custom-tags).
`x-litellm-num-retries`: Optional[int]: The number of retries for the request.
## Anthropic Headers
`anthropic-version` Optional[str]: The version of the Anthropic API to use.

1
litellm/proxy/_types.py
View File

@ -2780,6 +2780,7 @@ class LitellmDataForBackendLLMCall(TypedDict, total=False):
organization: str
timeout: Optional[float]
user: Optional[str]
num_retries: Optional[int]
class JWTKeyItem(TypedDict, total=False):

19
litellm/proxy/litellm_pre_call_utils.py
View File

@ -272,6 +272,16 @@ class LiteLLMProxyRequestSetup:
return float(timeout_header)
return None
@staticmethod
def _get_num_retries_from_request(headers: dict) -> Optional[int]:
"""
Workaround for client request from Vercel's AI SDK.
"""
num_retries_header = headers.get("x-litellm-num-retries", None)
if num_retries_header is not None:
return int(num_retries_header)
return None
@staticmethod
def _get_forwardable_headers(
headers: Union[Headers, dict],
@ -407,6 +417,10 @@ class LiteLLMProxyRequestSetup:
if timeout is not None:
data["timeout"] = timeout
num_retries = LiteLLMProxyRequestSetup._get_num_retries_from_request(headers)
if num_retries is not None:
data["num_retries"] = num_retries
return data
@staticmethod
@ -801,7 +815,10 @@ async def add_litellm_data_to_request( # noqa: PLR0915
data[k] = v
# Add disabled callbacks from key metadata
if user_api_key_dict.metadata and "litellm_disabled_callbacks" in user_api_key_dict.metadata:
if (
user_api_key_dict.metadata
and "litellm_disabled_callbacks" in user_api_key_dict.metadata
):
disabled_callbacks = user_api_key_dict.metadata["litellm_disabled_callbacks"]
if disabled_callbacks and isinstance(disabled_callbacks, list):
data["litellm_disabled_callbacks"] = disabled_callbacks

54
litellm/proxy/management_endpoints/common_daily_activity.py
View File

@ -80,6 +80,44 @@ def update_breakdown_metrics(
)
)
# Update model group breakdown
if record.model_group and record.model_group not in breakdown.model_groups:
breakdown.model_groups[record.model_group] = MetricWithMetadata(
metrics=SpendMetrics(),
metadata=model_metadata.get(record.model_group, {}),
)
if record.model_group:
breakdown.model_groups[record.model_group].metrics = update_metrics(
breakdown.model_groups[record.model_group].metrics, record
)
# Update API key breakdown for this model
if (
record.api_key
not in breakdown.model_groups[record.model_group].api_key_breakdown
):
breakdown.model_groups[record.model_group].api_key_breakdown[
record.api_key
] = KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=KeyMetadata(
key_alias=api_key_metadata.get(record.api_key, {}).get(
"key_alias", None
),
team_id=api_key_metadata.get(record.api_key, {}).get(
"team_id", None
),
),
)
breakdown.model_groups[record.model_group].api_key_breakdown[
record.api_key
].metrics = update_metrics(
breakdown.model_groups[record.model_group]
.api_key_breakdown[record.api_key]
.metrics,
record,
)
if record.mcp_namespaced_tool_name:
if record.mcp_namespaced_tool_name not in breakdown.mcp_servers:
breakdown.mcp_servers[record.mcp_namespaced_tool_name] = MetricWithMetadata(
@ -295,22 +333,6 @@ async def get_daily_activity(
take=page_size,
)
# # for 50% of the records, set the mcp_server_id to a random value
# mcp_server_dict = {"Zapier_Gmail_MCP", "Stripe_MCP"}
# import random
# for idx, record in enumerate(daily_spend_data):
# record = LiteLLM_DailyUserSpend(**record.model_dump())
# if random.random() < 0.5:
# record.mcp_server_id = random.choice(list(mcp_server_dict))
# record.model = None
# record.model_group = None
# record.prompt_tokens = 0
# record.completion_tokens = 0
# record.cache_read_input_tokens = 0
# record.cache_creation_input_tokens = 0
# daily_spend_data[idx] = record
# Get all unique API keys from the spend data
api_keys = set()
for record in daily_spend_data:

3
litellm/types/proxy/management_endpoints/common_daily_activity.py
View File

@ -61,6 +61,9 @@ class BreakdownMetrics(BaseModel):
models: Dict[str, MetricWithMetadata] = Field(
default_factory=dict
) # model -> {metrics, metadata}
model_groups: Dict[str, MetricWithMetadata] = Field(
default_factory=dict
) # model_group -> {metrics, metadata}
providers: Dict[str, MetricWithMetadata] = Field(
default_factory=dict
) # provider -> {metrics, metadata}

177
tests/test_litellm/proxy/test_litellm_pre_call_utils.py
View File

@ -243,15 +243,13 @@ async def test_add_litellm_data_to_request_disabled_callbacks():
api_key="test_api_key",
user_id="test_user_id",
org_id="test_org_id",
metadata={
"litellm_disabled_callbacks": ["langfuse", "langsmith", "datadog"]
}
metadata={"litellm_disabled_callbacks": ["langfuse", "langsmith", "datadog"]},
)
# Setup request data
data = {
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hello"}]
"messages": [{"role": "user", "content": "Hello"}],
}
# Setup proxy config
@ -262,7 +260,7 @@ async def test_add_litellm_data_to_request_disabled_callbacks():
data=data,
request=request_mock,
user_api_key_dict=user_api_key_dict,
proxy_config=proxy_config
proxy_config=proxy_config,
)
# Verify that litellm_disabled_callbacks was added to the request data
@ -298,15 +296,13 @@ async def test_add_litellm_data_to_request_disabled_callbacks_empty():
api_key="test_api_key",
user_id="test_user_id",
org_id="test_org_id",
metadata={
"litellm_disabled_callbacks": []
}
metadata={"litellm_disabled_callbacks": []},
)
# Setup request data
data = {
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hello"}]
"messages": [{"role": "user", "content": "Hello"}],
}
# Setup proxy config
@ -317,7 +313,7 @@ async def test_add_litellm_data_to_request_disabled_callbacks_empty():
data=data,
request=request_mock,
user_api_key_dict=user_api_key_dict,
proxy_config=proxy_config
proxy_config=proxy_config,
)
# Verify that litellm_disabled_callbacks is not added when empty
@ -352,13 +348,13 @@ async def test_add_litellm_data_to_request_disabled_callbacks_not_present():
api_key="test_api_key",
user_id="test_user_id",
org_id="test_org_id",
metadata={} # No litellm_disabled_callbacks
metadata={}, # No litellm_disabled_callbacks
)
# Setup request data
data = {
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hello"}]
"messages": [{"role": "user", "content": "Hello"}],
}
# Setup proxy config
@ -369,7 +365,7 @@ async def test_add_litellm_data_to_request_disabled_callbacks_not_present():
data=data,
request=request_mock,
user_api_key_dict=user_api_key_dict,
proxy_config=proxy_config
proxy_config=proxy_config,
)
# Verify that litellm_disabled_callbacks is not added when not present
@ -404,15 +400,13 @@ async def test_add_litellm_data_to_request_disabled_callbacks_invalid_type():
api_key="test_api_key",
user_id="test_user_id",
org_id="test_org_id",
metadata={
"litellm_disabled_callbacks": "not_a_list" # Should be a list
}
metadata={"litellm_disabled_callbacks": "not_a_list"}, # Should be a list
)
# Setup request data
data = {
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hello"}]
"messages": [{"role": "user", "content": "Hello"}],
}
# Setup proxy config
@ -423,7 +417,7 @@ async def test_add_litellm_data_to_request_disabled_callbacks_invalid_type():
data=data,
request=request_mock,
user_api_key_dict=user_api_key_dict,
proxy_config=proxy_config
proxy_config=proxy_config,
)
# Verify that litellm_disabled_callbacks is not added when invalid type
@ -460,16 +454,20 @@ async def test_add_litellm_data_to_request_disabled_callbacks_with_logging_setti
org_id="test_org_id",
metadata={
"logging": [
{"callback_name": "langfuse", "callback_type": "success", "callback_vars": {}}
{
"callback_name": "langfuse",
"callback_type": "success",
"callback_vars": {},
}
],
"litellm_disabled_callbacks": ["langsmith", "datadog"]
}
"litellm_disabled_callbacks": ["langsmith", "datadog"],
},
)
# Setup request data
data = {
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hello"}]
"messages": [{"role": "user", "content": "Hello"}],
}
# Setup proxy config
@ -480,7 +478,7 @@ async def test_add_litellm_data_to_request_disabled_callbacks_with_logging_setti
data=data,
request=request_mock,
user_api_key_dict=user_api_key_dict,
proxy_config=proxy_config
proxy_config=proxy_config,
)
# Verify that both logging settings and disabled callbacks are handled correctly
@ -500,12 +498,8 @@ def test_key_dynamic_logging_settings():
# Test with arize logging
key_with_arize = UserAPIKeyAuth(
api_key="test-key",
metadata={
"logging": [
{"callback_name": "arize", "callback_type": "success"}
]
},
team_metadata={}
metadata={"logging": [{"callback_name": "arize", "callback_type": "success"}]},
team_metadata={},
)
result = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings(key_with_arize)
assert result == [{"callback_name": "arize", "callback_type": "success"}]
@ -514,22 +508,22 @@ def test_key_dynamic_logging_settings():
key_with_langfuse = UserAPIKeyAuth(
api_key="test-key",
metadata={
"logging": [
{"callback_name": "langfuse", "callback_type": "success"}
]
"logging": [{"callback_name": "langfuse", "callback_type": "success"}]
},
team_metadata={}
team_metadata={},
)
result = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings(
key_with_langfuse
)
result = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings(key_with_langfuse)
assert result == [{"callback_name": "langfuse", "callback_type": "success"}]
# Test with no logging metadata
key_without_logging = UserAPIKeyAuth(
api_key="test-key",
metadata={},
team_metadata={}
api_key="test-key", metadata={}, team_metadata={}
)
result = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings(
key_without_logging
)
result = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings(key_without_logging)
assert result is None
@ -542,12 +536,12 @@ def test_team_dynamic_logging_settings():
api_key="test-key",
metadata={},
team_metadata={
"logging": [
{"callback_name": "arize", "callback_type": "failure"}
]
}
"logging": [{"callback_name": "arize", "callback_type": "failure"}]
},
)
result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(
key_with_team_arize
)
result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(key_with_team_arize)
assert result == [{"callback_name": "arize", "callback_type": "failure"}]
# Test with langfuse team logging
@ -555,21 +549,21 @@ def test_team_dynamic_logging_settings():
api_key="test-key",
metadata={},
team_metadata={
"logging": [
{"callback_name": "langfuse", "callback_type": "success"}
]
}
"logging": [{"callback_name": "langfuse", "callback_type": "success"}]
},
)
result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(
key_with_team_langfuse
)
result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(key_with_team_langfuse)
assert result == [{"callback_name": "langfuse", "callback_type": "success"}]
# Test with no team logging metadata
key_without_team_logging = UserAPIKeyAuth(
api_key="test-key",
metadata={},
team_metadata={}
api_key="test-key", metadata={}, team_metadata={}
)
result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(
key_without_team_logging
)
result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(key_without_team_logging)
assert result is None
@ -588,22 +582,21 @@ def test_get_dynamic_logging_metadata_with_arize_team_logging():
"callback_type": "success",
"callback_vars": {
"arize_api_key": "test_arize_api_key",
"arize_space_id": "test_arize_space_id"
}
"arize_space_id": "test_arize_space_id",
},
}
]
}
},
)
# Mock proxy_config (not used in this test path since we have team dynamic logging)
mock_proxy_config = MagicMock()
# Call the function
result = _get_dynamic_logging_metadata(
user_api_key_dict=user_api_key_dict,
proxy_config=mock_proxy_config
user_api_key_dict=user_api_key_dict, proxy_config=mock_proxy_config
)
# Verify the result
assert result is not None
assert isinstance(result, TeamCallbackMetadata)
@ -613,6 +606,68 @@ def test_get_dynamic_logging_metadata_with_arize_team_logging():
assert result.callback_vars["arize_space_id"] == "test_arize_space_id"
def test_get_num_retries_from_request():
"""
Test LiteLLMProxyRequestSetup._get_num_retries_from_request method
"""
# Test case 1: Header is present with valid integer string
headers_with_retries = {"x-litellm-num-retries": "3"}
result = LiteLLMProxyRequestSetup._get_num_retries_from_request(
headers_with_retries
)
assert result == 3
# Test case 2: Header is not present
headers_without_retries = {"Content-Type": "application/json"}
result = LiteLLMProxyRequestSetup._get_num_retries_from_request(
headers_without_retries
)
assert result is None
# Test case 3: Empty headers dictionary
empty_headers = {}
result = LiteLLMProxyRequestSetup._get_num_retries_from_request(empty_headers)
assert result is None
# Test case 4: Header present with zero value
headers_with_zero = {"x-litellm-num-retries": "0"}
result = LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_with_zero)
assert result == 0
# Test case 5: Header present with large number
headers_with_large_number = {"x-litellm-num-retries": "100"}
result = LiteLLMProxyRequestSetup._get_num_retries_from_request(
headers_with_large_number
)
assert result == 100
# Test case 6: Multiple headers with num retries header
headers_multiple = {
"Content-Type": "application/json",
"x-litellm-num-retries": "5",
"Authorization": "Bearer token",
}
result = LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_multiple)
assert result == 5
# Test case 7: Header present with invalid value (should raise ValueError when int() is called)
headers_with_invalid = {"x-litellm-num-retries": "invalid"}
with pytest.raises(ValueError):
LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_with_invalid)
# Test case 8: Header present with float string (should raise ValueError when int() is called)
headers_with_float = {"x-litellm-num-retries": "3.5"}
with pytest.raises(ValueError):
LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_with_float)
# Test case 9: Header present with negative number
headers_with_negative = {"x-litellm-num-retries": "-1"}
result = LiteLLMProxyRequestSetup._get_num_retries_from_request(
headers_with_negative
)
assert result == -1
def test_add_user_api_key_auth_to_request_metadata():
"""
Test that add_user_api_key_auth_to_request_metadata properly adds user API key authentication data to request metadata
@ -668,4 +723,4 @@ def test_add_user_api_key_auth_to_request_metadata():
# Verify original data is preserved
assert result["model"] == "gpt-3.5-turbo"
assert result["messages"] == [{"role": "user", "content": "Hello"}]
assert result["messages"] == [{"role": "user", "content": "Hello"}]

81
ui/litellm-dashboard/src/components/new_usage.tsx
View File

@ -86,6 +86,7 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
});
const [allTags, setAllTags] = useState<EntityList[]>([]);
const [modelViewType, setModelViewType] = useState<'groups' | 'individual'>('groups');
const getAllTags = async () => {
if (!accessToken) {
@ -160,6 +161,58 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
.slice(0, 5);
};
const getTopModelGroups = () => {
const modelGroupSpend: { [key: string]: MetricWithMetadata } = {};
userSpendData.results.forEach((day) => {
Object.entries(day.breakdown.model_groups || {}).forEach(([modelGroup, metrics]) => {
if (!modelGroupSpend[modelGroup]) {
modelGroupSpend[modelGroup] = {
metrics: {
spend: 0,
prompt_tokens: 0,
completion_tokens: 0,
total_tokens: 0,
api_requests: 0,
successful_requests: 0,
failed_requests: 0,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
},
metadata: {},
api_key_breakdown: {}
};
}
modelGroupSpend[modelGroup].metrics.spend += metrics.metrics.spend;
modelGroupSpend[modelGroup].metrics.prompt_tokens +=
metrics.metrics.prompt_tokens;
modelGroupSpend[modelGroup].metrics.completion_tokens +=
metrics.metrics.completion_tokens;
modelGroupSpend[modelGroup].metrics.total_tokens += metrics.metrics.total_tokens;
modelGroupSpend[modelGroup].metrics.api_requests += metrics.metrics.api_requests;
modelGroupSpend[modelGroup].metrics.successful_requests +=
metrics.metrics.successful_requests || 0;
modelGroupSpend[modelGroup].metrics.failed_requests +=
metrics.metrics.failed_requests || 0;
modelGroupSpend[modelGroup].metrics.cache_read_input_tokens +=
metrics.metrics.cache_read_input_tokens || 0;
modelGroupSpend[modelGroup].metrics.cache_creation_input_tokens +=
metrics.metrics.cache_creation_input_tokens || 0;
});
});
return Object.entries(modelGroupSpend)
.map(([modelGroup, metrics]) => ({
key: modelGroup,
spend: metrics.metrics.spend,
requests: metrics.metrics.api_requests,
successful_requests: metrics.metrics.successful_requests,
failed_requests: metrics.metrics.failed_requests,
tokens: metrics.metrics.total_tokens,
}))
.sort((a, b) => b.spend - a.spend)
.slice(0, 5);
};
// Calculate provider spend from the breakdown data
const getProviderSpend = () => {
const providerSpend: { [key: string]: MetricWithMetadata } = {};
@ -527,11 +580,35 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
<Col numColSpan={1}>
<Card className="h-full">
<div className="flex justify-between items-center mb-4">
<Title>Top Models</Title>
<Title>
{modelViewType === 'groups' ? 'Top Public Model Names' : 'Top Litellm Models'}
</Title>
<div className="flex bg-gray-100 rounded-lg p-1">
<button
className={`px-3 py-1 text-sm rounded-md transition-colors ${
modelViewType === 'groups'
? 'bg-white shadow-sm text-gray-900'
: 'text-gray-600 hover:text-gray-900'
}`}
onClick={() => setModelViewType('groups')}
>
Public Model Name
</button>
<button
className={`px-3 py-1 text-sm rounded-md transition-colors ${
modelViewType === 'individual'
? 'bg-white shadow-sm text-gray-900'
: 'text-gray-600 hover:text-gray-900'
}`}
onClick={() => setModelViewType('individual')}
>
Litellm Model Name
</button>
</div>
</div>
<BarChart
className="mt-4 h-40"
data={getTopModels()}
data={modelViewType === 'groups' ? getTopModelGroups() : getTopModels()}
index="key"
categories={["spend"]}
colors={["cyan"]}

115
ui/litellm-dashboard/src/components/usage/types.ts
View File

@ -1,88 +1,89 @@
export interface SpendMetrics {
spend: number;
prompt_tokens: number;
completion_tokens: number;
total_tokens: number;
api_requests: number;
successful_requests: number;
failed_requests: number;
cache_read_input_tokens: number;
cache_creation_input_tokens: number;
spend: number
prompt_tokens: number
completion_tokens: number
total_tokens: number
api_requests: number
successful_requests: number
failed_requests: number
cache_read_input_tokens: number
cache_creation_input_tokens: number
}
export interface DailyData {
date: string;
metrics: SpendMetrics;
breakdown: BreakdownMetrics;
date: string
metrics: SpendMetrics
breakdown: BreakdownMetrics
}
export interface BreakdownMetrics {
models: { [key: string]: MetricWithMetadata };
mcp_servers: { [key: string]: MetricWithMetadata };
providers: { [key: string]: MetricWithMetadata };
api_keys: { [key: string]: KeyMetricWithMetadata };
entities: { [key: string]: MetricWithMetadata };
models: { [key: string]: MetricWithMetadata }
model_groups: { [key: string]: MetricWithMetadata }
mcp_servers: { [key: string]: MetricWithMetadata }
providers: { [key: string]: MetricWithMetadata }
api_keys: { [key: string]: KeyMetricWithMetadata }
entities: { [key: string]: MetricWithMetadata }
}
export interface MetricWithMetadata {
metrics: SpendMetrics;
metadata: object;
api_key_breakdown: { [key: string]: KeyMetricWithMetadata };
metrics: SpendMetrics
metadata: object
api_key_breakdown: { [key: string]: KeyMetricWithMetadata }
}
export interface KeyMetricWithMetadata {
metrics: SpendMetrics;
metadata: KeyMetadata;
metrics: SpendMetrics
metadata: KeyMetadata
}
export interface KeyMetadata {
key_alias: string | null;
team_id: string | null;
key_alias: string | null
team_id: string | null
}
export interface TopApiKeyData {
api_key: string;
key_alias: string | null;
team_id: string | null;
spend: number;
requests: number;
tokens: number;
api_key: string
key_alias: string | null
team_id: string | null
spend: number
requests: number
tokens: number
}
export interface ModelActivityData {
label: string;
total_requests: number;
total_successful_requests: number;
total_failed_requests: number;
total_cache_read_input_tokens: number;
total_cache_creation_input_tokens: number;
total_tokens: number;
prompt_tokens: number;
completion_tokens: number;
total_spend: number;
top_api_keys: TopApiKeyData[];
label: string
total_requests: number
total_successful_requests: number
total_failed_requests: number
total_cache_read_input_tokens: number
total_cache_creation_input_tokens: number
total_tokens: number
prompt_tokens: number
completion_tokens: number
total_spend: number
top_api_keys: TopApiKeyData[]
daily_data: {
date: string;
date: string
metrics: {
prompt_tokens: number;
completion_tokens: number;
total_tokens: number;
api_requests: number;
spend: number;
successful_requests: number;
failed_requests: number;
cache_read_input_tokens: number;
cache_creation_input_tokens: number;
};
}[];
prompt_tokens: number
completion_tokens: number
total_tokens: number
api_requests: number
spend: number
successful_requests: number
failed_requests: number
cache_read_input_tokens: number
cache_creation_input_tokens: number
}
}[]
}
export interface EntityMetadata {
alias: string;
id: string;
alias: string
id: string
}
export interface EntityMetricWithMetadata {
metrics: SpendMetrics;
metadata: EntityMetadata;
metrics: SpendMetrics
metadata: EntityMetadata
}