.github/workflows/setup-observability-server-with-existed-vhost.yml: add mirrors.onwalk.net

.github/workflows/setup-observability-server-with-existed-vhost.yml

@@ -119,6 +119,7 @@ jobs:
           cat > hosts/inventory << EOF
           [master]
           k3s-dev              ansible_host=34.146.25.114
+          mirrors.onwalk.net   ansible_host=8.130.11.192
           
           [all:vars]
           ansible_port=22

playbook/roles/alerting/tasks/main.yml

@@ -1,4 +1,4 @@
-- name: get db password
+roles/alerting/tasks/main.yml- name: get db password
   shell: 'kubectl get secret --namespace database postgresql -o jsonpath="{.data.postgres-password}" | base64 -d'
   register: command_raw
   when: inventory_hostname in groups[group][0]

playbook/roles/alerting/templates/alerting_rules

@@ -0,0 +1,37 @@
+data:
+  alerting_rules.yml: |
+    groups:
+    - name: host-monitoring
+      rules:
+      - alert: HighLoad
+        expr: node_load1 > 2.0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: High load on {{ $labels.instance }}
+          description: "Load is {{ $value }} (threshold: 2.0)"
+      - alert: HighCpuUsage
+        expr: 100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: High CPU usage on {{ $labels.instance }}
+          description: "CPU usage is {{ $value }}%"
+      - alert: HighMemoryUsage
+        expr: (node_memory_MemTotal_bytes - node_memory_MemFree_bytes - node_memory_Buffers_bytes - node_memory_Cached_bytes) / node_memory_MemTotal_bytes * 100 > 90
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: High memory usage on {{ $labels.instance }}
+          description: "Memory usage is {{ $value }}%"
+      - alert: HighDiskUsage
+        expr: node_filesystem_avail_bytes{fstype="ext4"} / node_filesystem_size_bytes{fstype="ext4"} * 100 < 10
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: High disk usage on {{ $labels.instance }}
+          description: "Disk usage is {{ $value }}%

playbook/roles/alerting/templates/recording_rules

@@ -1,3 +1,16 @@
+data:
+  recording_rules.yml: |
+    groups:
+    - name: host-monitoring
+      rules:
+      - record: node_load1
+        expr: node_load1
+      - record: node_cpu_usage
+        expr: 100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)
+      - record: node_memory_usage
+        expr: (node_memory_MemTotal_bytes - node_memory_MemFree_bytes - node_memory_Buffers_bytes - node_memory_Cached_bytes) / node_memory_MemTotal_bytes * 100
+      - record: node_disk_usage
+        expr: 100 - (avg by (instance) (node_filesystem_avail_bytes{fstype="ext4"} / node_filesystem_size_bytes{fstype="ext4"}) * 100)
 groups:
 - name: 实例存活告警规则
   rules: