From 3f4835765254197ca5893d4f24a41b2d4537000e Mon Sep 17 00:00:00 2001
From: Haitao Pan <manbuzhe2009@qq.com>
Date: Fri, 26 May 2023 14:41:50 +0800
Subject: [PATCH] 
 .github/workflows/setup-observability-server-with-existed-vhost.yml: add
 mirrors.onwalk.net

---
 ...bservability-server-with-existed-vhost.yml |  1 +
 playbook/roles/alerting/tasks/main.yml        |  2 +-
 .../roles/alerting/templates/alerting_rules   | 37 +++++++++++++++++++
 .../roles/alerting/templates/recording_rules  | 13 +++++++
 4 files changed, 52 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/setup-observability-server-with-existed-vhost.yml b/.github/workflows/setup-observability-server-with-existed-vhost.yml
index 8624e51c..508bed41 100644
--- a/.github/workflows/setup-observability-server-with-existed-vhost.yml
+++ b/.github/workflows/setup-observability-server-with-existed-vhost.yml
@@ -119,6 +119,7 @@ jobs:
           cat > hosts/inventory << EOF
           [master]
           k3s-dev              ansible_host=34.146.25.114
+          mirrors.onwalk.net   ansible_host=8.130.11.192
           
           [all:vars]
           ansible_port=22
diff --git a/playbook/roles/alerting/tasks/main.yml b/playbook/roles/alerting/tasks/main.yml
index a239a37e..cd7f06e3 100755
--- a/playbook/roles/alerting/tasks/main.yml
+++ b/playbook/roles/alerting/tasks/main.yml
@@ -1,4 +1,4 @@
-- name: get db password
+roles/alerting/tasks/main.yml- name: get db password
   shell: 'kubectl get secret --namespace database postgresql -o jsonpath="{.data.postgres-password}" | base64 -d'
   register: command_raw
   when: inventory_hostname in groups[group][0]
diff --git a/playbook/roles/alerting/templates/alerting_rules b/playbook/roles/alerting/templates/alerting_rules
index e69de29b..6b8a01e9 100644
--- a/playbook/roles/alerting/templates/alerting_rules
+++ b/playbook/roles/alerting/templates/alerting_rules
@@ -0,0 +1,37 @@
+data:
+  alerting_rules.yml: |
+    groups:
+    - name: host-monitoring
+      rules:
+      - alert: HighLoad
+        expr: node_load1 > 2.0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: High load on {{ $labels.instance }}
+          description: "Load is {{ $value }} (threshold: 2.0)"
+      - alert: HighCpuUsage
+        expr: 100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: High CPU usage on {{ $labels.instance }}
+          description: "CPU usage is {{ $value }}%"
+      - alert: HighMemoryUsage
+        expr: (node_memory_MemTotal_bytes - node_memory_MemFree_bytes - node_memory_Buffers_bytes - node_memory_Cached_bytes) / node_memory_MemTotal_bytes * 100 > 90
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: High memory usage on {{ $labels.instance }}
+          description: "Memory usage is {{ $value }}%"
+      - alert: HighDiskUsage
+        expr: node_filesystem_avail_bytes{fstype="ext4"} / node_filesystem_size_bytes{fstype="ext4"} * 100 < 10
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: High disk usage on {{ $labels.instance }}
+          description: "Disk usage is {{ $value }}%
diff --git a/playbook/roles/alerting/templates/recording_rules b/playbook/roles/alerting/templates/recording_rules
index 04089c3e..061e1c5e 100644
--- a/playbook/roles/alerting/templates/recording_rules
+++ b/playbook/roles/alerting/templates/recording_rules
@@ -1,3 +1,16 @@
+data:
+  recording_rules.yml: |
+    groups:
+    - name: host-monitoring
+      rules:
+      - record: node_load1
+        expr: node_load1
+      - record: node_cpu_usage
+        expr: 100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)
+      - record: node_memory_usage
+        expr: (node_memory_MemTotal_bytes - node_memory_MemFree_bytes - node_memory_Buffers_bytes - node_memory_Cached_bytes) / node_memory_MemTotal_bytes * 100
+      - record: node_disk_usage
+        expr: 100 - (avg by (instance) (node_filesystem_avail_bytes{fstype="ext4"} / node_filesystem_size_bytes{fstype="ext4"}) * 100)
 groups:
 - name: 实例存活告警规则
   rules: