diff --git a/Dockerfile b/Dockerfile index 7c8bfb876a..fe57d94c8a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -67,10 +67,11 @@ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl # Generate prisma client RUN prisma generate RUN chmod +x docker/entrypoint.sh +RUN chmod +x docker/prod_entrypoint.sh EXPOSE 4000/tcp -ENTRYPOINT ["litellm"] +ENTRYPOINT ["docker/prod_entrypoint.sh"] # Append "--detailed_debug" to the end of CMD to view detailed debug logs CMD ["--port", "4000"] diff --git a/docker/Dockerfile.alpine b/docker/Dockerfile.alpine index 2cbebc38a5..70ab9cac01 100644 --- a/docker/Dockerfile.alpine +++ b/docker/Dockerfile.alpine @@ -48,8 +48,11 @@ COPY --from=builder /wheels/ /wheels/ # Install the built wheel using pip; again using a wildcard if it's the only file RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels +RUN chmod +x docker/entrypoint.sh +RUN chmod +x docker/prod_entrypoint.sh + EXPOSE 4000/tcp # Set your entrypoint and command -ENTRYPOINT ["litellm"] +ENTRYPOINT ["docker/prod_entrypoint.sh"] CMD ["--port", "4000"] diff --git a/docker/Dockerfile.custom_ui b/docker/Dockerfile.custom_ui index 7dee3c1f16..5a31314211 100644 --- a/docker/Dockerfile.custom_ui +++ b/docker/Dockerfile.custom_ui @@ -33,6 +33,7 @@ WORKDIR /app # Make sure your docker/entrypoint.sh is executable RUN chmod +x docker/entrypoint.sh +RUN chmod +x docker/prod_entrypoint.sh # Expose the necessary port EXPOSE 4000/tcp diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 96bfa9705d..5e660618a9 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -67,12 +67,12 @@ RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh # Generate prisma client RUN prisma generate RUN chmod +x docker/entrypoint.sh - +RUN chmod +x docker/prod_entrypoint.sh EXPOSE 4000/tcp # # Set your entrypoint and command -ENTRYPOINT ["litellm"] +ENTRYPOINT ["docker/prod_entrypoint.sh"] # Append "--detailed_debug" to the end of CMD to view detailed debug logs # CMD ["--port", "4000", "--detailed_debug"] diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 4f1a2dce33..f89e133877 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -71,13 +71,15 @@ RUN chmod -R 777 /.cache RUN pip install nodejs-bin RUN pip install prisma RUN prisma generate + RUN chmod +x docker/entrypoint.sh +RUN chmod +x docker/prod_entrypoint.sh EXPOSE 4000/tcp # # Set your entrypoint and command -ENTRYPOINT ["litellm"] +ENTRYPOINT ["docker/prod_entrypoint.sh"] # Append "--detailed_debug" to the end of CMD to view detailed debug logs # CMD ["--port", "4000", "--detailed_debug"] diff --git a/docker/prod_entrypoint.sh b/docker/prod_entrypoint.sh new file mode 100644 index 0000000000..077aa11ea2 --- /dev/null +++ b/docker/prod_entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +if [ "$USE_DDTRACE" = "true" ]; then + exec ddtrace-run litellm "$@" +else + exec litellm "$@" +fi \ No newline at end of file diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index cd89a756a4..c1bd894040 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -1070,6 +1070,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ LiteLLM Supports logging to the following Datdog Integrations: - `datadog` [Datadog Logs](https://docs.datadoghq.com/logs/) - `datadog_llm_observability` [Datadog LLM Observability](https://www.datadoghq.com/product/llm-observability/) +- `ddtrace-run` [Datadog Tracing](#datadog-tracing) @@ -1142,6 +1143,21 @@ Expected output on Datadog +#### Datadog Tracing + +Use `ddtrace-run` to enable [Datadog Tracing](https://ddtrace.readthedocs.io/en/stable/installation_quickstart.html) on litellm proxy + +Pass `USE_DDTRACE=true` to the docker run command. When `USE_DDTRACE=true`, the proxy will run `ddtrace-run litellm` as the `ENTRYPOINT` instead of just `litellm` + +```bash +docker run \ + -v $(pwd)/litellm_config.yaml:/app/config.yaml \ + -e USE_DDTRACE=true \ + -p 4000:4000 \ + ghcr.io/berriai/litellm:main-latest \ + --config /app/config.yaml --detailed_debug +``` + ### Set DD variables (`DD_SERVICE` etc) LiteLLM supports customizing the following Datadog environment variables diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index df074fd540..5890787bf1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -535,7 +535,13 @@ async def proxy_startup_event(app: FastAPI): proxy_batch_write_at=proxy_batch_write_at, proxy_logging_obj=proxy_logging_obj, ) + ## [Optional] Initialize dd tracer + ProxyStartupEvent._init_dd_tracer() + + # End of startup event yield + + # Shutdown event await proxy_shutdown_event() @@ -3218,6 +3224,19 @@ class ProxyStartupEvent: await prisma_client.health_check() return prisma_client + @classmethod + def _init_dd_tracer(cls): + """ + Initialize dd tracer - if `USE_DDTRACE=true` in .env + + DD tracer is used to trace Python applications. + Doc: https://docs.datadoghq.com/tracing/trace_collection/automatic_instrumentation/dd_libraries/python/ + """ + if get_secret_bool("USE_DDTRACE", False) is True: + import ddtrace + + ddtrace.patch_all(logging=True) + #### API ENDPOINTS #### @router.get( diff --git a/requirements.txt b/requirements.txt index 487ca49982..2f6137600e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,6 +19,7 @@ google-generativeai==0.5.0 # for vertex ai calls async_generator==1.10.0 # for async ollama calls langfuse==2.45.0 # for langfuse self-hosted logging prometheus_client==0.20.0 # for /metrics endpoint on proxy +ddtrace==2.19.0rc1 # for advanced DD tracing / profiling orjson==3.10.12 # fast /embedding responses apscheduler==3.10.4 # for resetting budget in background fastapi-sso==0.16.0 # admin UI, SSO