ai-workspace-infra/iac_modules
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add roles: deepflow-agent grafana-agent prometheus-transfer xtls-server

Browse Source
This commit is contained in:
Haitao Pan 2023-04-18 16:18:46 +08:00
parent f2c170c1b3
commit a451bd3c88
29 changed files with 413 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
playbook/roles/alerting/tasks/main.yml
View File

@ -12,6 +12,6 @@
script: files/setup-observable-server.sh {{ domain }} {{ secret }} {{ namespace }} {{ mysql_db_password }}
when: inventory_hostname in groups[group]
- name: Check alerting rules config
shell: promtool check rules /path/to/example.rules.yml
- name: Check alerting rules config
shell: promtool check rules /path/to/example.rules.yml
when: inventory_hostname in groups[group]

10
playbook/roles/clickhouse/tasks/main.yml
View File

@ -4,6 +4,16 @@
- name: Prep NameSpace
shell: "kubectl create namespace clickhouse || echo true"
- name: get clickhouse node ips
shell: "kubectl get pods -n clickhouse -o wide | grep clickhouse | awk '{print $6}'"
register: ck_node_ips_raw
when: inventory_hostname in groups[group][0]
- name: set fact join command for ck_node_ip1
set_fact:
ck_node_ip1 : "{{ ck_node_ips_raw.stdout_lines[0] }}"
when: inventory_hostname in groups[group][0]
- name: sync clickhouse deploy yaml
template: src=templates/{{ item }} dest=/tmp/{{ item }} owner=root group=root mode=0644 force=yes unsafe_writes=yes
with_items:

3
playbook/roles/clickhouse/templates/clickhouse-cluster/clickhouse-config.yaml
View File

@ -23,14 +23,12 @@ data:
<server_id from_env="CK_INDEX"/>
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
<coordination_settings>
<operation_timeout_ms>10000</operation_timeout_ms>
<session_timeout_ms>30000</session_timeout_ms>
<raft_logs_level>trace</raft_logs_level>
<rotate_log_storage_interval>10000</rotate_log_storage_interval>
</coordination_settings>
<raft_configuration>
<server>
<id>0</id>
@ -49,7 +47,6 @@ data:
</server>
</raft_configuration>
</keeper_server>
<zookeeper>
<node>
<host>clickhouse-0.clickhouse-service.clickhouse</host>

5
playbook/roles/clickhouse/templates/qryn/qryn-deployment.yaml
View File

@ -24,9 +24,9 @@ spec:
- name: CLICKHOUSE_AUTH
value: "default"
- name: CLICKHOUSE_PORT
value: "80"
value: "8123"
- name: CLICKHOUSE_SERVER
value: "clickhouse.onwalk.net"
value: "{{ ck_node_ip1 }}"
image: qxip/qryn
name: qryn
ports:
@ -34,4 +34,3 @@ spec:
resources: {}
restartPolicy: Always
status: {}

1
playbook/roles/common/tasks/main.yml
View File

@ -20,4 +20,3 @@
- name: Install packages
shell: "apt install -y auditd"
when: ansible_facts['distribution'] == "Ubuntu"

3
playbook/roles/common/templates/authorized_keys Executable file
View File

@ -0,0 +1,3 @@
{% for item in ssh_keys %}
{{ item }}
{% endfor %}

20
playbook/roles/deepflow-agent/tasks/main.yml Executable file
View File

@ -0,0 +1,20 @@
- name: Pre setting
shell: "rm -f /usr/bin/deepflow-agent; rm -f /tmp/deepflow-agent.tar.gz"
when: inventory_hostname in groups[group]
- name: Download prometheus binary
shell: 'curl -Lo /tmp/deepflow-agent.tar.gz https://deepflow-ce.oss-cn-beijing.aliyuncs.com/bin/agent/stable/linux/amd64/deepflow-agent.tar.gz && \
tar -zxvf /tmp/deepflow-agent.tar.gz -C /usr/sbin/'
when: inventory_hostname in groups[group]
- name: Create deepflow-agent service
template: src=templates/deepflow-agent.service dest=/etc/systemd/system/deepflow-agent.service owner=root group=root mode=0644
when: inventory_hostname in groups[group]
- name: Create prometheus-agent config
template: src=templates/deepflow-agent.yaml dest=/etc/deepflow-agent.yaml owner=root group=root mode=0644
when: inventory_hostname in groups[group]
- name: Init prometheus-agent service
shell: "systemctl enable deepflow-agent && systemctl daemon-reload && systemctl restart deepflow-agent"
when: inventory_hostname in groups[group]

14
playbook/roles/deepflow-agent/templates/deepflow-agent.service Normal file
View File

@ -0,0 +1,14 @@
[Unit]
Description=deepflow-agent.service
After=syslog.target network-online.target
[Service]
Environment=GOTRACEBACK=single
LimitCORE=1G
ExecStart=/usr/sbin/deepflow-agent
Restart=always
RestartSec=10
LimitNOFILE=1024:4096
[Install]
WantedBy=multi-user.target

3
playbook/roles/deepflow-agent/templates/deepflow-agent.yaml Normal file
View File

@ -0,0 +1,3 @@
controller-ips:
- {{ cluster_node }}
vtap-group-id-request: "{{ cluster_id }}"

11
playbook/roles/fluent-bit-agent/tasks/main.yml Executable file
View File

@ -0,0 +1,11 @@
- name: Install pkgs
shell: 'curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh'
when: inventory_hostname in groups[group]
- name: Create fluent-bit config
template: src=templates/fluent-bit.conf dest=/etc/fluent-bit/fluent-bit.conf owner=root group=root mode=0644
when: inventory_hostname in groups[group]
- name: Init fluent-bit service
shell: "systemctl enable fluent-bit && systemctl daemon-reload && systemctl restart fluent-bit"
when: inventory_hostname in groups[group]

57
playbook/roles/fluent-bit-agent/templates/fluent-bit.conf Normal file
View File

@ -0,0 +1,57 @@
[SERVICE]
flush 1
daemon Off
log_level info
parsers_file parsers.conf
plugins_file plugins.conf
http_server Off
http_listen 0.0.0.0
http_port 2020
storage.metrics on
# storage.path /tmp/storage
# storage.sync normal
# storage.checksum off
# storage.backlog.mem_limit 5M
[INPUT]
Name tail
Tag syslog
Path /var/log/syslog
Parser syslog-rfc5424
Mem_Buf_Limit 5MB
Skip_Long_Lines On
Refresh_Interval 10
[INPUT]
Name tail
Tag authlog
Path /var/log/auth.log
Parser syslog-rfc5424
Mem_Buf_Limit 5MB
Skip_Long_Lines On
Refresh_Interval 10
[INPUT]
Name systemd
Tag agent
Systemd_Filter _SYSTEMD_UNIT=prometheus-agent.service
[OUTPUT]
name loki
match syslog
host {{ loki_host }}
port {{ loki_port }}
labels {{ label }}
[OUTPUT]
name loki
match agent
host {{ loki_host }}
port {{ loki_port }}
labels job=agent
[OUTPUT]
name loki
match authlog
host {{ loki_host }}
port {{ loki_port }}
labels job=auth

2
playbook/roles/grafana-agent/meta/main.yml Normal file
View File

@ -0,0 +1,2 @@
dependencies:
- role: node-exporter

16
playbook/roles/grafana-agent/tasks/main.yml Executable file
View File

@ -0,0 +1,16 @@
- name: Install pkgs
shell: 'curl -Lo /tmp/grafana-agent-0.32.1-1.amd64.deb https://mirrors.onwalk.net/pkgs/grafana-agent-0.32.1-1.amd64.deb && \
dpkg -i /tmp/grafana-agent-0.32.1-1.amd64.deb && apt install -f'
when: inventory_hostname in groups[group] and ansible_facts['distribution'] = "Ubuntu"
- name: Install pkgs
shell: 'curl -Lo /tmp/grafana-agent-0.32.1-1.amd64.rpm https://mirrors.onwalk.net/pkgs/grafana-agent-0.32.1-1.amd64.rpm && \
yum install /tmp/grafana-agent-0.32.1-1.amd64.rpm'
when: inventory_hostname in groups[group] and ansible_facts['distribution'] != "Ubuntu"
- name: Create prometheus-agent config
template: src=templates/prometheus.yml dest=/etc/prometheus/prometheus.yml owner=oot group=root mode=0644
when: inventory_hostname in groups[group]
- name: Init grafana-agent service
shell: "systemctl enable grafana-agent && systemctl daemon-reload && systemctl restart grafana-agent"
when: inventory_hostname in groups[group]

22
playbook/roles/grafana-agent/templates/grafana-agent.yaml Normal file
View File

@ -0,0 +1,22 @@
integrations:
agent:
enabled: true
metrics:
wal_directory: /tmp/wal
global:
remote_write:
- url: http://localhost:9009/api/prom/push
loki:
configs:
- name: default
positions:
filename: /tmp/positions.yaml
scrape_configs:
- job_name: varlogs
static_configs:
- targets: [localhost]
labels:
job: varlogs
__path__: /var/log/*log
clients:
- url: http://qryn:3100/loki/api/v1/push

2
playbook/roles/k3s/files/setup-k3s-with-hostpath-sc.sh
View File

@ -16,7 +16,7 @@ case `uname -m` in
loongarch64) ARCH=loongarch64; ;;
*) echo "un-supported arch, exit ..."; exit 1; ;;
esac
rm -rf helm.tar.gz* /usr/local/bin/helm || echo true
rm -rf helm.tar.gz* /usr/local/bin/helm || echo true
sudo wget --no-check-certificate https://mirrors.onwalk.net/tools/linux-${ARCH}/helm.tar.gz && sudo tar -xvpf helm.tar.gz -C /usr/local/bin/
sudo chmod 755 /usr/local/bin/helm

5
playbook/roles/keycloak/files/setup-keycloak.sh
View File

@ -28,6 +28,7 @@ externalDatabase:
password: "$keycloak_db_password"
EOF
helm repo add bitnami https://charts.bitnami.com/bitnami || echo true
helm repo add stable https://artifact.onwalk.net/chartrepo/public/ || echo true
helm repo update
helm upgrade --install keycloak bitnami/keycloak -n $namespace -f keycloak-values.yaml
kubectl create ns ${namespace} || echo true
helm upgrade --install keycloak stable/keycloak -n $namespace -f keycloak-values.yaml

11
playbook/roles/keycloak/tasks/main.yml
View File

@ -1,5 +1,5 @@
- name: get db password
shell: 'kubectl get secret --namespace database postgresql -o jsonpath="{.data.postgres-password}" | base64 -d'
shell: 'kubectl get secret --namespace database postgresql -o jsonpath="{.data.postgres-password}" | base64 -d'
register: command_raw
when: inventory_hostname in groups[group][0]
@ -14,3 +14,12 @@
- name: Setup Keycloak Server
script: files/setup-keycloak.sh {{ hostvars[groups[group][0]].keycloak_db_password }} {{ admin_password }} {{ domain }} {{ secret }} {{ namespace }}
when: inventory_hostname in groups[group]
- name: Sync aws-gloabl-oidc-broker deploy yaml
template: src=templates/{{ item }} dest=/tmp/{{ item }} owner=root group=root mode=0644 force=yes unsafe_writes=yes
with_items:
- aws-gloabl-oidc-broker.yaml
- name: Setup aws-gloabl-oidc-broker
shell: "kubectl apply -f /tmp/aws-gloabl-oidc-broker.yaml"
when: inventory_hostname in groups[group]

74
playbook/roles/keycloak/templates/aws-gloabl-oidc-broker.yaml Normal file
View File

@ -0,0 +1,74 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: aws-global-oidc-broker
namespace: itsm
labels:
io.metrico.service: aws-global-oidc-broker
spec:
replicas: 1
selector:
matchLabels:
io.metrico.service: aws-global-oidc-broker
strategy: {}
template:
metadata:
creationTimestamp: null
labels:
io.metrico.service: aws-global-oidc-broker
spec:
containers:
- name: aws-global-oidc-broker
image: artifact.onwalk.net/public/aws-global-oidc-broker:1.2.0
ports:
- containerPort: 5000
resources: {}
env:
- name: KEYCLOAK_CLIENT_ID
value: "aws-oidc"
- name: KEYCLOAK_WELLKNOWN
value: "https://keycloak.apollo-ev.com/realms/cloud-sso/.well-known/openid-configuration"
- name: KEYCLOAK_CLIENT_SECRET
value: "WYyZJGUOOiwooIp700PtykmjYkrsPJPi"
- name: TITLE
value: "aws-oidc"
restartPolicy: Always
---
apiVersion: v1
kind: Service
metadata:
creationTimestamp: null
name: aws-global-oidc-broker
namespace: itsm
spec:
ports:
- name: "5000"
port: 5000
nodePort: 5000
targetPort: 5000
selector:
io.metrico.service: aws-global-oidc-broker
type: NodePort
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: aws-global-oidc-broker
namespace: itsm
spec:
ingressClassName: nginx
rules:
- host: loki.apollo-ev.com
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: aws-global-oidc-broker
port:
number: 5000
tls:
- hosts:
- loki.apollo-ev.com
secretName: keycloak-tls

2
playbook/roles/mysql/tasks/main.yml
View File

@ -1,5 +1,5 @@
- name: Setup MySQL Server
script: files/setup-mysql.sh
script: files/setup-mysql.sh
when: inventory_hostname in groups[group]
- name: Post Setup MySQL Server

4
playbook/roles/openldap/files/setup-openldap.sh
View File

@ -38,7 +38,7 @@ ltb-passwd:
- openldap-ltb.${domain}
EOF
helm repo add helm-openldap https://jp-gouin.github.io/helm-openldap/
helm repo add stable https://artifact.onwalk.net/chartrepo/public/
helm repo up
kubectl create ns ${namespace} || echo true
helm upgrade --install openldap helm-openldap/openldap-stack-ha -n ${namespace} --create-namespace -f openldap-vaules.yaml
helm upgrade --install openldap stable/openldap-stack-ha -n ${namespace} --create-namespace -f openldap-vaules.yaml

2
playbook/roles/prometheus-transfer/meta/main.yml Normal file
View File

@ -0,0 +1,2 @@
dependencies:
- role: node-exporter

21
playbook/roles/prometheus-transfer/tasks/main.yml Executable file
View File

@ -0,0 +1,21 @@
- name: Pre setting
shell: "rm -f /usr/bin/prometheus; mkdir -pv /opt/prometheus/data/ && chown prometheus:prometheus /opt/prometheus/data/"
- name: download prometheus binary
shell: 'curl -Lo /usr/bin/prometheus https://mirrors.onwalk.net/tools/linux-amd64/prometheus && chmod 755 /usr/bin/prometheus'
- name: create prometheus-transfer service
template: src=templates/prometheus-transfer.service dest=/etc/systemd/system/prometheus-transfer.service owner=root group=root mode=0644
- name: create prometheus-transfer config
template: src=templates/prometheus-transfer.yml dest=/etc/prometheus/prometheus-transfer.yml owner=root group=root mode=0644
- name: create prometheus-transfer start script
template: src=templates/start-prometheus-transfer-service.sh dest=/usr/bin/start-prometheus-transfer-service.sh owner=root group=root mode=0755
- name: create prometheus-transfer stop script
template: src=templates/stop-prometheus-transfer-service.sh dest=/usr/bin/stop-prometheus-transfer-service.sh owner=root group=root mode=0755
- name: init prometheus-transfer service
shell: "systemctl enable prometheus-transfer && systemctl daemon-reload && systemctl restart prometheus-transfer"

16
playbook/roles/prometheus-transfer/templates/prometheus-transfer.service Normal file
View File

@ -0,0 +1,16 @@
[Unit]
Description=Prometheus
Documentation=https://prometheus.io/
After=network.target
[Service]
Type=simple
User=prometheus
ExecStart=/usr/bin/start-prometheus-transfer-service.sh
ExecStop=/usr/bin/stop-prometheus-transfer-service.sh
Restart=on-failure
RestartSec=30
StartLimitInterval=0
[Install]
WantedBy=multi-user.target

9
playbook/roles/prometheus-transfer/templates/prometheus-transfer.yml Normal file
View File

@ -0,0 +1,9 @@
global:
scrape_interval: 3s
evaluation_interval: 3s
remote_read:
- url: '{{ remote_read }}'
read_recent: true
basic_auth:
username: '{{ remote_user }}'
password: '{{ remote_token }}'

2
playbook/roles/prometheus-transfer/templates/start-prometheus-transfer-service.sh Executable file
View File

@ -0,0 +1,2 @@
#!/bin/sh
/usr/bin/prometheus --config.file=/etc/prometheus/prometheus-transfer.yml --web.listen-address="0.0.0.0:9092" --web.enable-lifecycle --storage.tsdb.path="/opt/prometheus/data/"

2
playbook/roles/prometheus-transfer/templates/stop-prometheus-transfer-service.sh Executable file
View File

@ -0,0 +1,2 @@
#!/bin/sh
pkill -9 prometheus

2
playbook/roles/redis/tasks/main.yml
View File

@ -1,3 +1,3 @@
- name: Setup Redis Server
script: files/setup-redis.sh
script: files/setup-redis.sh
when: inventory_hostname in groups[group]

23
playbook/roles/xtls-server/tasks/main.yml Executable file
View File

@ -0,0 +1,23 @@
- name: Remove fluent-bit service
shell: "systemctl disable fluent-bit && systemctl stop fluent-bit"
when: inventory_hostname in groups[group]
- name: Pre setting
shell: "rm -f /usr/bin/promtail; mkdir -pv /etc/promtail/ && touch /var/log/positions.yaml"
when: inventory_hostname in groups[group]
- name: Download promtail binary
shell: 'curl -Lo /usr/bin/promtail https://mirrors.onwalk.net/tools/linux-amd64/promtail && chmod 755 /usr/bin/promtail'
when: inventory_hostname in groups[group]
- name: Create promtail-agent service
template: src=templates/promtail-agent.service dest=/lib/systemd/system/promtail-agent.service owner=root group=root mode=0644
when: inventory_hostname in groups[group]
- name: Create promtail-agent config
template: src=templates/promtail.yaml dest=/etc/promtail/promtail.yaml owner=root group=root mode=0644
when: inventory_hostname in groups[group]
- name: Init promtail-agent service
shell: "systemctl enable promtail-agent && systemctl daemon-reload && systemctl restart promtail-agent"
when: inventory_hostname in groups[group]

84
playbook/roles/xtls-server/templates/config.json Normal file
View File

@ -0,0 +1,84 @@
{
"log": {
"loglevel": "warning"
},
"routing": {
"domainStrategy": "IPIfNonMatch",
"rules": [
{
"type": "field",
"ip": [
"geoip:cn"
],
"outboundTag": "block"
}
]
},
"inbounds": [
{
"listen": "0.0.0.0",
"port": 1443,
"protocol": "vless",
"settings": {
"clients": [
{
"id": "18d270a9-533d-4b13-b3f1-e7f55540a9b2",
"flow": "xtls-rprx-vision"
}
],
"decryption": "none",
"fallbacks": [
{
"dest": "8001",
"xver": 1
},
{
"alpn": "h2",
"dest": "8002",
"xver": 1
}
]
},
"streamSettings": {
"network": "tcp",
"security": "tls",
"tlsSettings": {
"rejectUnknownSni": true,
"minVersion": "1.2",
"certificates": [
{
"ocspStapling": 3600,
"certificateFile": "/etc/ssl/onwalk.net.pem",
"keyFile": "/etc/ssl/onwalk.net.key"
}
]
}
},
"sniffing": {
"enabled": true,
"destOverride": [
"http",
"tls"
]
}
}
],
"outbounds": [
{
"protocol": "freedom",
"tag": "direct"
},
{
"protocol": "blackhole",
"tag": "block"
}
],
"policy": {
"levels": {
"0": {
"handshake": 2,
"connIdle": 120
}
}
}
}