chart: fix ovs-ovn upgrade (#3164)

Signed-off-by: 张祖建 <zhangzujian.7@gmail.com>
kubeovn · Sep 4, 2023 · f3b6460 · f3b6460
1 parent ff16355
commit f3b6460
Show file tree

Hide file tree

Showing 8 changed files with 106 additions and 57 deletions.
diff --git a/.github/workflows/scheduled-e2e.yaml b/.github/workflows/scheduled-e2e.yaml
@@ -25,7 +25,6 @@ jobs:
         branch:
           - master
           - release-1.11
-          - release-1.10
           - release-1.9
           - release-1.8
         ip-family:
@@ -115,7 +114,6 @@ jobs:
         branch:
           - master
           - release-1.11
-          - release-1.10
           - release-1.9
           - release-1.8
         ip-family:
@@ -198,7 +196,6 @@ jobs:
         branch:
           - master
           - release-1.11
-          - release-1.10
           - release-1.9
           - release-1.8
         ip-family:
@@ -281,7 +278,6 @@ jobs:
         branch:
           - master
           - release-1.11
-          - release-1.10
           - release-1.9
           - release-1.8
         ip-family:
@@ -341,7 +337,6 @@ jobs:
         branch:
           - master
           - release-1.11
-          - release-1.10
           - release-1.9
           - release-1.8
         ip-family:
@@ -425,7 +420,6 @@ jobs:
         branch:
           - master
           - release-1.11
-          - release-1.10
           - release-1.9
     steps:
       - uses: actions/checkout@v3
@@ -547,7 +541,6 @@ jobs:
         branch:
           - master
           - release-1.11
-          - release-1.10
           - release-1.9
     steps:
       - uses: actions/checkout@v3
@@ -598,7 +591,6 @@ jobs:
         branch:
           - master
           - release-1.11
-          - release-1.10
           - release-1.9
           - release-1.8
     steps:
@@ -652,7 +644,6 @@ jobs:
         branch:
           - master
           - release-1.11
-          - release-1.10
           - release-1.9
           - release-1.8
     steps:
@@ -918,7 +909,6 @@ jobs:
         branch:
           - master
           - release-1.11
-          - release-1.10
           - release-1.9
           - release-1.8
     steps:
@@ -969,7 +959,6 @@ jobs:
         branch:
           - master
           - release-1.11
-          - release-1.10
           - release-1.9
           - release-1.8
     steps:
@@ -1057,7 +1046,6 @@ jobs:
         branch:
           - master
           - release-1.11
-          - release-1.10
           - release-1.9
         ssl:
           - "true"
@@ -1148,8 +1136,11 @@ jobs:
       matrix:
         case:
           - release-1.9 => release-1.11
+          - release-1.9 => release-1.12
           - release-1.9 => master
+          - release-1.11 => release-1.12
           - release-1.11 => master
+          - release-1.12 => master
     steps:
       - uses: actions/checkout@v3
       - uses: azure/setup-helm@v3
@@ -1211,7 +1202,7 @@ jobs:
         run: |
           sudo pip3 install j2cli
           sudo pip3 install "j2cli[yaml]"
-          sudo PATH=~/.local/bin:$PATH make kind-init
+          sudo PATH=~/.local/bin:$PATH make kind-init-ha
           sudo cp -r /root/.kube/ ~/.kube/
           sudo chown -R $(id -un). ~/.kube/
 
@@ -1227,16 +1218,7 @@ jobs:
         run: |
           version=$(grep -E '^VERSION="v([0-9]+\.){2}[0-9]+"$' dist/images/install.sh | head -n1 | awk -F= '{print $2}' | tr -d '"')
           docker pull kubeovn/kube-ovn:$version
-
-          restart_ovs=false
-          v1=$(printf "$VERSION_FROM\\nrelease-1.11" | sort -Vr | head -n1)
-          v2=$(printf "$VERSION_TO\\nrelease-1.12" | sort -Vr | head -n1)
-          if [ $v1 = "release-1.11" ]; then
-            if [ $VERSION_TO = "master" -o $VERSION_TO = $v2 ]; then
-              restart_ovs=true
-            fi
-          fi
-          CHART_UPGRADE_RESTART_OVS=$restart_ovs VERSION=$version make kind-upgrade-chart
+          VERSION=$version make kind-upgrade-chart
 
       - name: Run E2E
         env:

diff --git a/Makefile b/Makefile
@@ -20,8 +20,6 @@ endif
 
 CONTROL_PLANE_TAINTS = node-role.kubernetes.io/master node-role.kubernetes.io/control-plane
 
-CHART_UPGRADE_RESTART_OVS=$(shell echo $${CHART_UPGRADE_RESTART_OVS:-false})
-
 MULTUS_VERSION = v4.0.2
 MULTUS_IMAGE = ghcr.io/k8snetworkplumbingwg/multus-cni:$(MULTUS_VERSION)-thick
 MULTUS_YAML = https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/$(MULTUS_VERSION)/deployments/multus-daemonset-thick.yml
@@ -423,30 +421,31 @@ kind-install-chart: kind-load-image kind-untaint-control-plane
 	kubectl label node -lbeta.kubernetes.io/os=linux kubernetes.io/os=linux --overwrite
 	kubectl label node -lnode-role.kubernetes.io/control-plane kube-ovn/role=master --overwrite
 	kubectl label node -lovn.kubernetes.io/ovs_dp_type!=userspace ovn.kubernetes.io/ovs_dp_type=kernel --overwrite
-	ips=$$(kubectl get node -lkube-ovn/role=master --no-headers -o wide | awk '{print $$6}') && \
+	ips=$$(kubectl get node -lkube-ovn/role=master --no-headers -o wide | awk '{print $$6}' | tr '\n' ',' | sed 's/,$$//') && \
 	helm install kubeovn ./charts \
 		--set global.images.kubeovn.tag=$(VERSION) \
-		--set replicaCount=$$(echo $$ips | awk '{print NF}') \
-		--set MASTER_NODES="$$(echo $$ips | tr \\n ',' | sed -e 's/,$$//' -e 's/,/\\,/g')"
-	kubectl rollout status deployment/ovn-central -n kube-system --timeout 300s
-	kubectl rollout status deployment/kube-ovn-controller -n kube-system --timeout 120s
-	kubectl rollout status daemonset/kube-ovn-cni -n kube-system --timeout 120s
-	kubectl rollout status daemonset/kube-ovn-pinger -n kube-system --timeout 120s
-	kubectl rollout status deployment/coredns -n kube-system --timeout 60s
+		--set replicaCount=$$(echo $$ips | awk -F ',' '{print NF}') \
+		--set MASTER_NODES="$$(echo $$ips | sed 's/,/\\,/g')"
+	sleep 60
+	kubectl -n kube-system rollout status --timeout=1s deployment/ovn-central
+	kubectl -n kube-system rollout status --timeout=1s daemonset/ovs-ovn
+	kubectl -n kube-system rollout status --timeout=1s deployment/kube-ovn-controller
+	kubectl -n kube-system rollout status --timeout=1s daemonset/kube-ovn-cni
+	kubectl -n kube-system rollout status --timeout=1s daemonset/kube-ovn-pinger
 
 .PHONY: kind-upgrade-chart
 kind-upgrade-chart: kind-load-image
-	$(eval OVN_DB_IPS = $(shell kubectl get no -lkube-ovn/role=master --no-headers -o wide | awk '{print $$6}' | tr \\n ',' | sed -e 's/,$$//' -e 's/,/\\,/g'))
+	$(eval OVN_DB_IPS = $(shell kubectl get node -lkube-ovn/role=master --no-headers -o wide | awk '{print $$6}' | tr '\n' ',' | sed -e 's/,$$//' -e 's/,/\\,/g'))
 	helm upgrade kubeovn ./charts \
 		--set global.images.kubeovn.tag=$(VERSION) \
 		--set replicaCount=$$(echo $(OVN_DB_IPS) | awk -F ',' '{print NF}') \
-		--set MASTER_NODES='$(OVN_DB_IPS)' \
-		--set restart_ovs=$(CHART_UPGRADE_RESTART_OVS)
-	kubectl rollout status deployment/ovn-central -n kube-system --timeout 300s
-	kubectl rollout status daemonset/ovs-ovn -n kube-system --timeout 120s
-	kubectl rollout status deployment/kube-ovn-controller -n kube-system --timeout 120s
-	kubectl rollout status daemonset/kube-ovn-cni -n kube-system --timeout 120s
-	kubectl rollout status daemonset/kube-ovn-pinger -n kube-system --timeout 120s
+		--set MASTER_NODES='$(OVN_DB_IPS)'
+	sleep 90
+	kubectl -n kube-system rollout status --timeout=1s deployment/ovn-central
+	kubectl -n kube-system rollout status --timeout=1s daemonset/ovs-ovn
+	kubectl -n kube-system rollout status --timeout=1s deployment/kube-ovn-controller
+	kubectl -n kube-system rollout status --timeout=1s daemonset/kube-ovn-cni
+	kubectl -n kube-system rollout status --timeout=1s daemonset/kube-ovn-pinger
 
 .PHONY: kind-install
 kind-install: kind-load-image

diff --git a/charts/README.md b/charts/README.md
@@ -18,9 +18,3 @@ $ helm install --debug kubeovn ./kubeovn-helm --set MASTER_NODES=${Node0},${Node
 # upgrade to this version
 $ helm upgrade --debug kubeovn ./kubeovn-helm --set MASTER_NODES=${Node0},${Node1},${Node2}, --set replicaCount=3
 ```
-
-If you are upgrading Kube-OVN from versions prior to v1.12, you need to set `restart_ovs` to `true`:
-
-```shell
-$ helm upgrade --debug kubeovn ./kubeovn-helm --set MASTER_NODES=${Node0},${Node1},${Node2}, --set replicaCount=3 --set restart_ovs=true
-```
diff --git a/charts/templates/post-upgrade.yaml → charts/templates/upgrade-ovs-ovn.yaml b/charts/templates/post-upgrade.yaml → charts/templates/upgrade-ovs-ovn.yaml
@@ -1,4 +1,72 @@
-{{ if .Values.restart_ovs }}
+{{ if (lookup "apps/v1" "DaemonSet" "kube-system" "ovs-ovn") }}
+{{ if eq (lookup "apps/v1" "DaemonSet" "kube-system" "ovs-ovn").spec.updateStrategy.type "OnDelete" }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ovs-ovn-upgrade
+  namespace: kube-system
+  annotations:
+    # This is what defines this resource as a hook. Without this line, the
+    # job is considered part of the release.
+    "helm.sh/hook": post-upgrade
+    "helm.sh/hook-weight": "1"
+    "helm.sh/hook-delete-policy": hook-succeeded
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    rbac.authorization.k8s.io/system-only: "true"
+    # This is what defines this resource as a hook. Without this line, the
+    # job is considered part of the release.
+    "helm.sh/hook": post-upgrade
+    "helm.sh/hook-weight": "2"
+    "helm.sh/hook-delete-policy": hook-succeeded
+  name: system:ovs-ovn-upgrade
+rules:
+  - apiGroups:
+      - apps
+    resources:
+      - daemonsets
+    resourceNames:
+      - ovs-ovn
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - list
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - list
+      - get
+      - delete
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: ovs-ovn-upgrade
+  annotations:
+    # This is what defines this resource as a hook. Without this line, the
+    # job is considered part of the release.
+    "helm.sh/hook": post-upgrade
+    "helm.sh/hook-weight": "3"
+    "helm.sh/hook-delete-policy": hook-succeeded
+roleRef:
+  name: system:ovs-ovn-upgrade
+  kind: ClusterRole
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+  - kind: ServiceAccount
+    name: ovs-ovn-upgrade
+    namespace: kube-system
+---
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -13,7 +81,7 @@ metadata:
     # This is what defines this resource as a hook. Without this line, the
     # job is considered part of the release.
     "helm.sh/hook": post-upgrade
-    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-weight": "4"
     "helm.sh/hook-delete-policy": hook-succeeded
 spec:
   completions: 1
@@ -49,8 +117,8 @@ spec:
       hostNetwork: true
       nodeSelector:
         kubernetes.io/os: "linux"
-      serviceAccount: ovn
-      serviceAccountName: ovn
+      serviceAccount: ovs-ovn-upgrade
+      serviceAccountName: ovs-ovn-upgrade
       containers:
         - name: post-upgrade-job
           image: "{{ .Values.global.registry.address}}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }}"
@@ -71,3 +139,4 @@ spec:
           hostPath:
             path: /var/log/kube-ovn
 {{ end }}
+{{ end }}
diff --git a/charts/values.yaml b/charts/values.yaml
@@ -114,8 +114,6 @@ imagePullSecrets: []
 nameOverride: ""
 fullnameOverride: ""
 
-restart_ovs: false
-
 # hybrid dpdk
 HYBRID_DPDK: false
 HUGEPAGE_SIZE_TYPE: hugepages-2Mi # Default 

diff --git a/dist/images/start-ovs.sh b/dist/images/start-ovs.sh
@@ -55,7 +55,14 @@ function quit {
     pid=$(/usr/share/ovn/scripts/ovn-ctl status_controller | awk '{print $NF}')
     if cgroup_match "${pid}" self; then
       /usr/share/ovn/scripts/grace_stop_ovn_controller
-      /usr/share/openvswitch/scripts/ovs-ctl stop
+    fi
+    pid=$(/usr/share/openvswitch/scripts/ovs-ctl status | grep ovsdb-server | awk '{print $NF}')
+    if cgroup_match "${pid}" self; then
+      /usr/share/openvswitch/scripts/ovs-ctl --no-ovs-vswitchd stop
+    fi
+    pid=$(/usr/share/openvswitch/scripts/ovs-ctl status | grep ovs-vswitchd | awk '{print $NF}')
+    if cgroup_match "${pid}" self; then
+      /usr/share/openvswitch/scripts/ovs-ctl --no-ovsdb-server stop
     fi
   fi
 

diff --git a/dist/images/upgrade-ovs.sh b/dist/images/upgrade-ovs.sh
@@ -5,9 +5,9 @@ set -e
 POD_NAMESPACE=${POD_NAMESPACE:-kube-system}
 
 dsGenVer=`kubectl -n $POD_NAMESPACE get ds ovs-ovn -o jsonpath={.metadata.generation}`
+kubectl -n $POD_NAMESPACE delete pod -l app=ovs,pod-template-generation!=$dsGenVer
+
 for node in `kubectl get node -o jsonpath='{.items[*].metadata.name}'`; do
-  # delete pod with old version
-  kubectl -n $POD_NAMESPACE delete pod -l app=ovs,pod-template-generation!=$dsGenVer --field-selector spec.nodeName=$node
   # wait the pod with new version to be created and delete it
   while true; do
     pod=`kubectl -n $POD_NAMESPACE get pod -l app=ovs,pod-template-generation=$dsGenVer --field-selector spec.nodeName=$node -o name`

diff --git a/yamls/ovn.yaml b/yamls/ovn.yaml
@@ -214,7 +214,7 @@ spec:
   updateStrategy:
     type: RollingUpdate
     rollingUpdate:
-      maxSurge: 100%
+      maxSurge: 1
       maxUnavailable: 0
   template:
     metadata: