From 3cc809b2105155c9bef615c1070b3670ce5edbb2 Mon Sep 17 00:00:00 2001 From: Haitao Pan Date: Mon, 29 May 2023 19:32:57 +0800 Subject: [PATCH] roles/k3s: update --- playbook/jobs/init_k3s_cluster | 6 +- playbook/jobs/init_k3s_cluster_ali | 5 +- .../files/setup-ingress-apisix-controller.sh | 18 --- ...etup-ingress-nginx.sh => setup-ingress.sh} | 29 ++++- .../k3s/files/setup-k3s-with-custom-cni.sh | 45 ------- playbook/roles/k3s/files/setup-k3s.sh | 112 +++++++++++++----- playbook/roles/k3s/tasks/main.yml | 19 ++- 7 files changed, 125 insertions(+), 109 deletions(-) delete mode 100644 playbook/roles/k3s/files/setup-ingress-apisix-controller.sh rename playbook/roles/k3s/files/{setup-ingress-nginx.sh => setup-ingress.sh} (55%) delete mode 100644 playbook/roles/k3s/files/setup-k3s-with-custom-cni.sh diff --git a/playbook/jobs/init_k3s_cluster b/playbook/jobs/init_k3s_cluster index 848f1049..530ab31d 100644 --- a/playbook/jobs/init_k3s_cluster +++ b/playbook/jobs/init_k3s_cluster @@ -12,9 +12,9 @@ version: 'v1.24.7+k3s1' ingress: nginx cni: default - pod_cidr: '10.17.0.0/16' - pod_gateway: '10.17.0.1' - svc_cidr: '10.97.0.0/16' + pod_cidr: '10.16.0.0/16' + pod_gateway: '10.16.0.1' + svc_cidr: '10.96.0.0/16' join_cidr: '100.64.0.0/16' cni_iface: 'eth0' cni_tunnel: 'geneve' diff --git a/playbook/jobs/init_k3s_cluster_ali b/playbook/jobs/init_k3s_cluster_ali index fa5fe949..e07834f8 100644 --- a/playbook/jobs/init_k3s_cluster_ali +++ b/playbook/jobs/init_k3s_cluster_ali @@ -8,9 +8,12 @@ name: k3s vars: group: master + cluster_domain: 'cluster.local' + version: 'v1.24.7+k3s1' + ingress: nginx + cni: kubeovn pod_cidr: '10.17.0.0/16' pod_gateway: '10.17.0.1' - node_ip: '10.255.0.2' svc_cidr: '10.97.0.0/16' join_cidr: '100.64.0.0/16' cni_iface: 'eth0' diff --git a/playbook/roles/k3s/files/setup-ingress-apisix-controller.sh b/playbook/roles/k3s/files/setup-ingress-apisix-controller.sh deleted file mode 100644 index 52590ed4..00000000 --- a/playbook/roles/k3s/files/setup-ingress-apisix-controller.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -export ingress_ip=$1 -helm repo add apisix https://charts.apiseven.com || echo true -helm repo update -kubectl create ns ingress || echo true -helm upgrade --install apisix apisix/apisix \ - --set etcd.replicaCount=1 \ - --set gateway.type=NodePort \ - --set gateway.http.enabled=true \ - --set gateway.http.nodePort=80 \ - --set gateway.tls.enabled=true \ - --set gateway.tls.nodePort=443 \ - --set gateway.externalIPs[0]=$ingress_ip \ - --set ingress-controller.enabled=true \ - --namespace ingress \ - --set ingress-controller.config.apisix.serviceNamespace=ingress \ - --kubeconfig /etc/rancher/k3s/k3s.yaml -kubectl get service --namespace ingress diff --git a/playbook/roles/k3s/files/setup-ingress-nginx.sh b/playbook/roles/k3s/files/setup-ingress.sh similarity index 55% rename from playbook/roles/k3s/files/setup-ingress-nginx.sh rename to playbook/roles/k3s/files/setup-ingress.sh index 7afb0a18..e75d4e5b 100644 --- a/playbook/roles/k3s/files/setup-ingress-nginx.sh +++ b/playbook/roles/k3s/files/setup-ingress.sh @@ -1,6 +1,8 @@ #!/bin/bash -ip=$1 +ingress=$1 +ingress_ip=$2 +if [[ $ingress_type == nginx ]]; then cat > value.yaml < nginx-cm.yaml << EOF @@ -20,7 +22,7 @@ metadata: name: nginx-nginx-ingress namespace: ingress data: - external-status-address: $ip + external-status-address: $ingress_ip proxy-connect-timeout: 10s proxy-read-timeout: 10s client-header-buffer-size: 64k @@ -54,3 +56,24 @@ kubectl create namespace ingress || echo true helm upgrade --install nginx nginx-stable/nginx-ingress --version=0.15.0 --namespace ingress -f value.yaml kubectl apply -f nginx-cm.yaml kubectl patch svc nginx-nginx-ingress -n ingress --patch-file nginx-svc-patch.yaml + +elif [[ $ingress_type == apisix ]]; then + +export ingress_ip=$1 +helm repo add apisix https://charts.apiseven.com || echo true +helm repo update +kubectl create ns ingress || echo true +helm upgrade --install apisix apisix/apisix \ + --set etcd.replicaCount=1 \ + --set gateway.type=NodePort \ + --set gateway.http.enabled=true \ + --set gateway.http.nodePort=80 \ + --set gateway.tls.enabled=true \ + --set gateway.tls.nodePort=443 \ + --set gateway.externalIPs[0]=$ingress_ip \ + --set ingress-controller.enabled=true \ + --namespace ingress \ + --set ingress-controller.config.apisix.serviceNamespace=ingress \ + --kubeconfig /etc/rancher/k3s/k3s.yaml +kubectl get service --namespace ingress +fi diff --git a/playbook/roles/k3s/files/setup-k3s-with-custom-cni.sh b/playbook/roles/k3s/files/setup-k3s-with-custom-cni.sh deleted file mode 100644 index 14e8d1df..00000000 --- a/playbook/roles/k3s/files/setup-k3s-with-custom-cni.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -export version=$1 -export cluster_domain=$2 -mkdir -pv /opt/rancher/k3s - -ping -c 1 google.com > /dev/null 2>&1 -if [ $? -eq 0 ]; then - echo "当前主机在国际网络上" - curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=$version sh -s - \ - --disable=traefik,servicelb \ - --cluster-domain=$cluster_domain \ - --write-kubeconfig-mode 644 \ - --flannel-backend=none \ - --disable-network-policy \ - --write-kubeconfig ~/.kube/config \ - --data-dir=/opt/rancher/k3s \ - --kube-apiserver-arg service-node-port-range=0-50000 -else - echo "当前主机在大陆网络上" - curl -sfL https://rancher-mirror.rancher.cn/k3s/k3s-install.sh | INSTALL_K3S_VERSION=$version INSTALL_K3S_MIRROR=cn sh -s - \ - --disable=traefik,servicelb \ - --cluster-domain=$cluster_domain \ - --flannel-backend=none \ - --disable-network-policy \ - --write-kubeconfig-mode 644 \ - --write-kubeconfig ~/.kube/config \ - --data-dir=/opt/rancher/k3s \ - --kube-apiserver-arg service-node-port-range=0-50000 -fi - -export KUBECONFIG=/etc/rancher/k3s/k3s.yaml - -case `uname -m` in - x86_64) ARCH=amd64; ;; - aarch64) ARCH=arm64; ;; - loongarch64) ARCH=loongarch64; ;; - *) echo "un-supported arch, exit ..."; exit 1; ;; -esac -rm -rf helm.tar.gz* /usr/local/bin/helm || echo true -sudo wget --no-check-certificate https://mirrors.onwalk.net/tools/linux-${ARCH}/helm.tar.gz && sudo tar -xvpf helm.tar.gz -C /usr/local/bin/ -sudo chmod 755 /usr/local/bin/helm - -helm repo add artifact https://artifact.onwalk.net/chartrepo/public/ | echo true -helm repo up diff --git a/playbook/roles/k3s/files/setup-k3s.sh b/playbook/roles/k3s/files/setup-k3s.sh index 224c2987..546a49e1 100644 --- a/playbook/roles/k3s/files/setup-k3s.sh +++ b/playbook/roles/k3s/files/setup-k3s.sh @@ -2,40 +2,94 @@ export version=$1 export cluster_domain=$2 -mkdir -pv /opt/rancher/k3s +export cni=$3 +export ingress=$4 +export svc_cidr=$5 -ping -c 1 google.com > /dev/null 2>&1 -if [ $? -eq 0 ]; then - echo "当前主机在国际网络上" - curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=$version sh -s - \ - --disable=traefik,servicelb \ - --cluster-domain=$cluster_domain \ - --write-kubeconfig-mode 644 \ - --write-kubeconfig ~/.kube/config \ - --data-dir=/opt/rancher/k3s \ - --kube-apiserver-arg service-node-port-range=0-50000 -else - echo "当前主机在大陆网络上" - curl -sfL https://rancher-mirror.rancher.cn/k3s/k3s-install.sh | INSTALL_K3S_VERSION=$version INSTALL_K3S_MIRROR=cn sh -s - \ +function set_k3s_default() +{ + mkdir -pv /opt/rancher/k3s + + ping -c 1 google.com > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo "当前主机在国际网络上" + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=$version sh -s - \ + --disable=traefik,servicelb \ + --cluster-domain=$cluster_domain \ + --service-cidr=$svc_cidr \ + --write-kubeconfig-mode 644 \ + --write-kubeconfig ~/.kube/config \ + --data-dir=/opt/rancher/k3s \ + --kube-apiserver-arg service-node-port-range=0-50000 + else + echo "当前主机在大陆网络上" + curl -sfL https://rancher-mirror.rancher.cn/k3s/k3s-install.sh | INSTALL_K3S_VERSION=$version INSTALL_K3S_MIRROR=cn sh -s - \ + --disable=traefik,servicelb \ + --cluster-domain=$cluster_domain \ + --service-cidr=$svc_cidr \ + --write-kubeconfig-mode 644 \ + --write-kubeconfig ~/.kube/config \ + --data-dir=/opt/rancher/k3s \ + --kube-apiserver-arg service-node-port-range=0-50000 + fi +} + +function set_k3s_cni_kubevon() +{ + mkdir -pv /opt/rancher/k3s + + ping -c 1 google.com > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo "当前主机在国际网络上" + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=$version sh -s - \ --disable=traefik,servicelb \ --cluster-domain=$cluster_domain \ + --service-cidr=$svc_cidr \ + --write-kubeconfig-mode 644 \ + --flannel-backend=none \ + --disable-network-policy \ + --write-kubeconfig ~/.kube/config \ + --data-dir=/opt/rancher/k3s \ + --kube-apiserver-arg service-node-port-range=0-50000 + else + echo "当前主机在大陆网络上" + curl -sfL https://rancher-mirror.rancher.cn/k3s/k3s-install.sh | INSTALL_K3S_VERSION=$version INSTALL_K3S_MIRROR=cn sh -s - \ + --disable=traefik,servicelb \ + --cluster-domain=$cluster_domain \ + --service-cidr=$svc_cidr \ + --flannel-backend=none \ + --disable-network-policy \ --write-kubeconfig-mode 644 \ --write-kubeconfig ~/.kube/config \ --data-dir=/opt/rancher/k3s \ --kube-apiserver-arg service-node-port-range=0-50000 + fi + +} + +funtion setup_helm() +{ + export KUBECONFIG=/etc/rancher/k3s/k3s.yaml + case `uname -m` in + x86_64) ARCH=amd64; ;; + aarch64) ARCH=arm64; ;; + loongarch64) ARCH=loongarch64; ;; + *) echo "un-supported arch, exit ..."; exit 1; ;; + esac + rm -rf helm.tar.gz* /usr/local/bin/helm || echo true + sudo wget --no-check-certificate https://mirrors.onwalk.net/tools/linux-${ARCH}/helm.tar.gz && sudo tar -xvpf helm.tar.gz -C /usr/local/bin/ + sudo chmod 755 /usr/local/bin/helm + + helm repo add artifact https://artifact.onwalk.net/chartrepo/public/ | echo true + helm repo up +} + +## __main__ ## +# +if [[ $cni == 'default' ]]; then + set_k3s_default +elif [[ $cni == 'kubeovn' ]]; then + set_k3s_cni_kubevon fi - -export KUBECONFIG=/etc/rancher/k3s/k3s.yaml - -case `uname -m` in - x86_64) ARCH=amd64; ;; - aarch64) ARCH=arm64; ;; - loongarch64) ARCH=loongarch64; ;; - *) echo "un-supported arch, exit ..."; exit 1; ;; -esac -rm -rf helm.tar.gz* /usr/local/bin/helm || echo true -sudo wget --no-check-certificate https://mirrors.onwalk.net/tools/linux-${ARCH}/helm.tar.gz && sudo tar -xvpf helm.tar.gz -C /usr/local/bin/ -sudo chmod 755 /usr/local/bin/helm - -helm repo add artifact https://artifact.onwalk.net/chartrepo/public/ | echo true -helm repo up + setup_helm +# end diff --git a/playbook/roles/k3s/tasks/main.yml b/playbook/roles/k3s/tasks/main.yml index 789273e5..25d9df71 100755 --- a/playbook/roles/k3s/tasks/main.yml +++ b/playbook/roles/k3s/tasks/main.yml @@ -1,23 +1,22 @@ - name: Setup K3S Server - script: files/setup-k3s.sh {{ version }} {{ cluster_domain }} + script: files/setup-k3s.sh {{ version }} {{ cluster_domain }} {{ cni }} {{ ingress }} {{ svc_cidr }} when: inventory_hostname in groups[group] - name: Setup K3S Ingress - script: files/setup-ingress-nginx.sh {{ ingress_ip }} + script: files/setup-ingress.sh {{ ingress }} {{ ingress_ip }} when: inventory_hostname in groups[group] - name: Setup DNS Provider script: files/setup-dns-provider.sh {{ dns_ak }} {{ dns_sk }} when: inventory_hostname in groups[group] -#- name: Sync K3S CNI Config -# template: src=templates/cni_install.sh dest=/tmp/ owner=root group=root mode=0644 -# when: inventory_hostname in groups[group] -# -#- name: Setup K3S CNI -# shell: 'bash /tmp/cni_install.sh' -# when: inventory_hostname in groups[group] -# ignore_errors: yes +- name: Sync K3S CNI Config + template: src=templates/cni_install.sh dest=/tmp/ owner=root group=root mode=0644 + when: ( inventory_hostname in groups[group] ) and (cni == kubeovn ) +- name: Setup K3S CNI + shell: 'bash /tmp/cni_install.sh' + when: ( inventory_hostname in groups[group] ) and (cni == kubeovn ) + ignore_errors: yes #- name: Setup Egress # script: files/setup-egress.sh {{ egress_ip }} {{ namespace }}