From 5652b0df738207ea9d46d6a271cbd41d1d72c18b Mon Sep 17 00:00:00 2001
From: Haitao Pan <manbuzhe2009@qq.com>
Date: Wed, 29 Mar 2023 16:58:01 +0800
Subject: [PATCH] add k3s over wireguard support

---
 playbook/jobs/init_k3s_cluster                |  1 +
 playbook/roles/common/tasks/main.yml          |  3 +++
 playbook/roles/k3s/files/setup-egress.sh      |  2 ++
 .../k3s/files/setup-k3s-with-hostpath-sc.sh   | 24 ++++++++++++-------
 playbook/roles/k3s/tasks/main.yml             |  2 +-
 5 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/playbook/jobs/init_k3s_cluster b/playbook/jobs/init_k3s_cluster
index 94322e33..de44a888 100644
--- a/playbook/jobs/init_k3s_cluster
+++ b/playbook/jobs/init_k3s_cluster
@@ -8,3 +8,4 @@
         name: k3s
       vars:
         group: master
+        namespace: default
diff --git a/playbook/roles/common/tasks/main.yml b/playbook/roles/common/tasks/main.yml
index 5a298f93..2552fe0a 100755
--- a/playbook/roles/common/tasks/main.yml
+++ b/playbook/roles/common/tasks/main.yml
@@ -10,6 +10,9 @@
 - name: Update /etc/hosts
   template: src=templates/hosts dest=/etc/hosts owner=root group=root mode=0644 force=yes unsafe_writes=yes
 
+- name: enable ip_forward
+  shell: 'echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.conf; echo "net.ipv4.conf.all.proxy_arp = 1" >> /etc/sysctl.conf ; sysctl -p /etc/sysctl.conf'
+
 - name: Install packages
   shell: "yum makecache && yum install -y audit"
   when: ansible_facts['distribution'] != "Ubuntu"
diff --git a/playbook/roles/k3s/files/setup-egress.sh b/playbook/roles/k3s/files/setup-egress.sh
index e578f092..998b079f 100644
--- a/playbook/roles/k3s/files/setup-egress.sh
+++ b/playbook/roles/k3s/files/setup-egress.sh
@@ -1,5 +1,6 @@
 #!/bin/bash
 ip=$1
+namespace=$2
 
 cat > /tmp/egress.yaml << EOF
 apiVersion: cilium.io/v2
@@ -11,6 +12,7 @@ spec:
   - podSelector:
       matchLabels:
         role: egress-gateway
+        io.kubernetes.pod.namespace: $namespace
   destinationCIDRs:
   - "0.0.0.0/0"
   egressGateway:
diff --git a/playbook/roles/k3s/files/setup-k3s-with-hostpath-sc.sh b/playbook/roles/k3s/files/setup-k3s-with-hostpath-sc.sh
index 8de0d648..892d8e63 100644
--- a/playbook/roles/k3s/files/setup-k3s-with-hostpath-sc.sh
+++ b/playbook/roles/k3s/files/setup-k3s-with-hostpath-sc.sh
@@ -6,6 +6,9 @@ curl -sfL https://get.k3s.io | sh -s - \
 	--disable-kube-proxy                                 \
 	--write-kubeconfig-mode 644                          \
 	--write-kubeconfig ~/.kube/config                    \
+	--node-external-ip 42.xx.xx.12                       \
+	--advertise-address 42.xx.xx.12                      \
+	--node-ip 192.168.1.1                                \
 	--data-dir=/opt/rancher/k3s                          \
 	--kube-apiserver-arg service-node-port-range=0-50000
 
@@ -21,13 +24,18 @@ rm -rf helm.tar.gz* /usr/local/bin/helm || echo true
 sudo wget --no-check-certificate https://mirrors.onwalk.net/tools/linux-${ARCH}/helm.tar.gz && sudo tar -xvpf helm.tar.gz -C /usr/local/bin/
 sudo chmod 755 /usr/local/bin/helm
 
-helm install cilium cilium/cilium --version 1.13.1 \
-	--namespace kube-system           \
-	--set egressGateway.enabled=true  \
-	--set bpf.masquerade=true         \
-	--set kubeProxyReplacement=strict \
-	--set operator.replicas=1 \
-	--set l7Proxy=false
-
+helm repo add cilium https://helm.cilium.io/
 helm repo add artifact https://artifact.onwalk.net/chartrepo/k8s/ | echo true
 helm repo up
+
+helm install cilium cilium/cilium --version 1.13.1 \
+	--namespace kube-system             \
+	--set bpf.masquerade=true           \
+	--set egressGateway.enabled=true    \
+	--set kubeProxyReplacement=strict   \
+	--set operator.replicas=1           \
+	--set devices=eth0                  \
+	--set k8sServiceHost=192.168.31.254 \
+	--set k8sServicePort=6443           \
+	--set ipv4NativeRoutingCIDR=192.168.31.254/32 \
+	--set l7Proxy=false
diff --git a/playbook/roles/k3s/tasks/main.yml b/playbook/roles/k3s/tasks/main.yml
index c13ac04e..a3584001 100755
--- a/playbook/roles/k3s/tasks/main.yml
+++ b/playbook/roles/k3s/tasks/main.yml
@@ -7,7 +7,7 @@
   when: inventory_hostname in groups[group]
 
 - name: Setup Egress
-  script: files/setup-egress.sh {{ egress_ip }}
+  script: files/setup-egress.sh {{ egress_ip }} {{ namespace }}
   when: inventory_hostname in groups[group]
 
 - name: Setup DNS Provider