#!/bin/bash set -e cat << EOF > values-custom.yaml deepflowServerNodeIPS: - 10.50.1.111 #deepflowK8sClusterID: "fffffff" # FIXME: K8s ClusterID image: repository: hub.deepflow.yunshan.net/public/deepflow-agent pullPolicy: Always tag: v6.5 EOF helm repo add deepflow https://deepflowio.github.io/deepflow helm repo update deepflow # use `helm repo update` when helm < 3.7.0 helm install deepflow-agent -n deepflow deepflow/deepflow-agent --create-namespace -f values-custom.yaml ######################################################################################################## helm repo add grafana https://grafana.github.io/helm-charts helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm repo update kubectl create ns deepflow || true helm upgrade --install kube-state-metrics prometheus-community/kube-state-metrics \ --namespace deepflow --create-namespace helm upgrade --install node-exporter prometheus-community/prometheus-node-exporter \ --namespace deepflow --create-namespace \ --set service.type=ClusterIP \ --set service.port=9100 cat > grafana-agent-values.yaml << EOF global: image: registry: "images.onwalk.net/public" agent: mode: 'static' configMap: create: true content: '' logs: enabled: false traces: enabled: false EOF helm upgrade --install grafana-agent grafana/grafana-agent --namespace deepflow -f grafana-agent-values.yaml cat > grafana-agent-configmap.yaml << EOF apiVersion: v1 data: config.yaml: |- server: log_level: info log_format: logfmt metrics: global: scrape_interval: 1m configs: - name: agent scrape_configs: - job_name: kube-state-metrics static_configs: - targets: ['10.43.155.169:8080'] - job_name: node-metrics static_configs: - targets: ['10.43.68.133:9100'] remote_write: - url: http://deepflow-agent.deepflow.svc.cluster.local/api/v1/prometheus kind: ConfigMap metadata: annotations: meta.helm.sh/release-name: grafana-agent meta.helm.sh/release-namespace: deepflow labels: app.kubernetes.io/instance: grafana-agent app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: grafana-agent app.kubernetes.io/version: v0.42.0 helm.sh/chart: grafana-agent-0.42.0 name: grafana-agent namespace: deepflow EOF kubectl apply -f grafana-agent-configmap.yaml kubectl get pods -n deepflow ######################################################################################################## helm repo add vector https://helm.vector.dev helm repo update cat << EOF > vector-values-custom.yaml role: Agent #nodeSelector: # allow/vector: "false" # resources -- Set Vector resource requests and limits. resources: requests: cpu: 200m memory: 256Mi limits: cpu: 200m memory: 256Mi image: repository: images.onwalk.net/public/timberio/vector pullPolicy: Always tag: "0.37.1-distroless-libc" podLabels: vector.dev/exclude: "true" app: deepflow # extraVolumes -- Additional Volumes to use with Vector Pods. # extraVolumes: # - name: opt-log # hostPath: # path: "/opt/log/" # extraVolumeMounts -- Additional Volume to mount into Vector Containers. # extraVolumeMounts: # - name: opt-log # mountPath: "/opt/log/" # readOnly: true customConfig: ## The configuration comes from https://vector.dev/docs/reference/configuration/global-options/#data_dir data_dir: /vector-data-dir api: enabled: true address: 127.0.0.1:8686 playground: false sources: kubernetes_logs: type: kubernetes_logs namespace_annotation_fields: namespace_labels: "" node_annotation_fields: node_labels: "" pod_annotation_fields: pod_annotations: "" pod_labels: "" transforms: remap_kubernetes_logs: type: remap inputs: - kubernetes_logs source: |- # try to parse json if is_string(.message) && is_json(string!(.message)) { tags = parse_json(.message) ?? {} .message = tags.message # FIXME: the log content key inside json del(tags.message) .json = tags } if !exists(.level) { if exists(.json) { .level = .json.level del(.json.level) } else { # match log levels surround by ``[]`` or ``<>`` with ignore case level_tags = parse_regex(.message, r'[\[\\\<](?(?i)INFOR?(MATION)?|WARN(ING)?|DEBUG?|ERROR?|TRACE|FATAL|CRIT(ICAL)?)[\]\\\>]') ?? {} if !exists(level_tags.level) { # match log levels surround by whitespace, required uppercase strictly in case mismatching level_tags = parse_regex(.message, r'[\s](?INFOR?(MATION)?|WARN(ING)?|DEBUG?|ERROR?|TRACE|FATAL|CRIT(ICAL)?)[\s]') ?? {} } if exists(level_tags.level) { level_tags.level = upcase(string!(level_tags.level)) .level = level_tags.level } } } if !exists(._df_log_type) { # default log type ._df_log_type = "user" } if !exists(.app_service) { # FIXME: files 模块没有此字段,请通过日志内容注入应用名称 .app_service = .kubernetes.container_name } sinks: http: encoding: codec: json inputs: - remap_kubernetes_logs # NOTE: 注意这里数据源是 transform 模块的 key type: http uri: http://deepflow-agent.deepflow/api/v1/log EOF helm upgrade --install vector vector/vector --namespace deepflow --create-namespace -f vector-values-custom.yaml