underlay: fix link name exchange (#2516)

kubeovn · Mar 22, 2023 · 8f5c808 · 8f5c808
1 parent 90dcd00
commit 8f5c808
Show file tree

Hide file tree

Showing 6 changed files with 167 additions and 272 deletions.
diff --git a/dist/images/start-ovs.sh b/dist/images/start-ovs.sh
@@ -72,194 +72,37 @@ ovsdb_server_ctl="/var/run/openvswitch/ovsdb-server.$(cat /var/run/openvswitch/o
 ovs-appctl -t "$ovsdb_server_ctl" vlog/set jsonrpc:file:err
 ovs-appctl -t "$ovsdb_server_ctl" vlog/set reconnect:file:err
 
-function exchange_link_names() {
-  mappings=($(ovs-vsctl --if-exists get open . external-ids:ovn-bridge-mappings | tr -d '"' | tr ',' ' '))
-  bridges=($(ovs-vsctl --no-heading --columns=name find bridge external-ids:vendor=kube-ovn external-ids:exchange-link-name=true))
+function handle_underlay_bridges() {
+  bridges=($(ovs-vsctl --no-heading --columns=name find bridge external-ids:vendor=kube-ovn))
   for br in ${bridges[@]}; do
-    provider=""
-    for m in ${mappings[*]}; do
-      if echo $m | grep -q ":$br"'$'; then
-        provider=${m%:$br}
-        break
-      fi
-    done
-    if [ "x$provider" = "x" ]; then
-      echo "error: failed to get provider name for bridge $br"
-      continue
-    fi
-
-    port="br-$provider"
-    if ip link show $port 2>/dev/null; then
-      echo "link $port already exists"
-      continue
-    fi
-    if ! ip link show $br 2>/dev/null; then
-      echo "link $br does not exists"
-      continue
+    if ! ip link show $br >/dev/null; then
+      # the bridge does not exist, leave it to be handled by kube-ovn-cni
+      echo "deleting ovs bridge $br"
+      ovs-vsctl --no-wait del-br $br
     fi
-
-    echo "change link name from $br to $port"
-    ipv4_routes=($(ip -4 route show dev $br | tr ' ' '#'))
-    ipv6_routes=($(ip -6 route show dev $br | tr ' ' '#'))
-    ip link set $br down
-    ip link set $br name $port
-    ip link set $port up
-
-    # transfer IPv4 routes
-    default_ipv4_routes=()
-    for route in ${ipv4_routes[@]}; do
-      r=$(echo $route | tr '#' ' ')
-      if echo $r | grep -q -w 'scope link'; then
-        printf "add/replace IPv4 route $r to $port\n"
-        ip -4 route replace $r dev $port
-      else
-        default_ipv4_routes=(${default_ipv4_routes[@]} $route)
-      fi
-    done
-    for route in ${default_ipv4_routes[@]}; do
-      r=$(echo $route | tr '#' ' ')
-      printf "add/replace IPv4 route $r to $port\n"
-      ip -4 route replace $r dev $port
-    done
-
-    # transfer IPv6 routes
-    default_ipv6_routes=()
-    for route in ${ipv6_routes[@]}; do
-      r=$(echo $route | tr '#' ' ')
-      if echo $r | grep -q -w 'scope link'; then
-        printf "add/replace IPv6 route $r to $port\n"
-        ip -6 route replace $r dev $port
-      else
-        default_ipv6_routes=(${default_ipv6_routes[@]} $route)
-      fi
-    done
-    for route in ${default_ipv6_routes[@]}; do
-      r=$(echo $route | tr '#' ' ')
-      printf "add/replace IPv6 route $r to $port\n"
-      ip -6 route replace $r dev $port
-    done
   done
-}
-
-exchange_link_names
 
-function wait_flows_pre_check() {
-  local devices=""
-  local ips=($(echo $OVN_DB_IPS | sed 's/,/ /g'))
-  for ip in ${ips[*]}; do
-    devices="$devices $(ip route get $ip | grep -oE 'dev .+' | awk '{print $2}')"
-  done
-
-  bridges=($(ovs-vsctl --no-heading --columns=name find bridge external-ids:vendor=kube-ovn))
+  bridges=($(ovs-vsctl --no-heading --columns=name find bridge external-ids:vendor=kube-ovn external-ids:exchange-link-name=true))
   for br in ${bridges[@]}; do
-    ports=($(ovs-vsctl list-ports $br))
-    for port in ${ports[@]}; do
-      if ! echo $devices | grep -qw "$port"; then
-        continue
-      fi
-
-      port_type=$(ovs-vsctl --no-heading --columns=type find interface name=$port)
-      if [ ! "x$port_type" = 'x""' ]; then
-        continue
-      fi
-
-      if ! ip link show $port | grep -qw "master ovs-system"; then
-        return 1
-      fi
-    done
+    if [ -z $(ip link show $br type openvswitch) ]; then
+      # the bridge does not exist, leave it to be handled by kube-ovn-cni
+      echo "deleting ovs bridge $br"
+      ovs-vsctl --no-wait del-br $br
+    fi
   done
-
-  return 0
 }
 
-skip_wait_flows=0
-if ! wait_flows_pre_check; then
-  skip_wait_flows=1
-fi
+handle_underlay_bridges
 
-if [ $skip_wait_flows -eq 0 ]; then
-  # When ovs-vswitchd starts with this value set as true, it will neither flush or
-  # expire previously set datapath flows nor will it send and receive any
-  # packets to or from the datapath. Please check ovs-vswitchd.conf.db.5.txt
-  ovs-vsctl --no-wait set open_vswitch . other_config:flow-restore-wait="true"
-else
-  ovs-vsctl --no-wait set open_vswitch . other_config:flow-restore-wait="false"
-fi
+# When ovs-vswitchd starts with this value set as true, it will neither flush or
+# expire previously set datapath flows nor will it send and receive any
+# packets to or from the datapath. Please check ovs-vswitchd.conf.db.5.txt
+ovs-vsctl --no-wait set open_vswitch . other_config:flow-restore-wait="true"
 
 # Start vswitchd. restart will automatically set/unset flow-restore-wait which is not what we want
 /usr/share/openvswitch/scripts/ovs-ctl start --no-ovsdb-server --system-id=random --no-mlockall
 /usr/share/openvswitch/scripts/ovs-ctl --protocol=udp --dport=6081 enable-protocol
 
-sleep 1
-
-function handle_underlay_bridges() {
-    bridges=($(ovs-vsctl --no-heading --columns=name find bridge external-ids:vendor=kube-ovn))
-    for br in ${bridges[@]}; do
-        echo "handle bridge $br"
-        ip link set $br up
-
-        ports=($(ovs-vsctl list-ports $br))
-        for port in ${ports[@]}; do
-            port_type=$(ovs-vsctl --no-heading --columns=type find interface name=$port)
-            if [ ! "x$port_type" = 'x""' ]; then
-              continue
-            fi
-
-            echo "handle port $port on bridge $br"
-            ipv4_routes=($(ip -4 route show dev $port | tr ' ' '#'))
-            ipv6_routes=($(ip -6 route show dev $port | tr ' ' '#'))
-
-            set +o pipefail
-            addresses=($(ip addr show dev $port | grep -E '^\s*inet[6]?\s+' | grep -w global | awk '{print $2}'))
-            set -o pipefail
-
-            # transfer IP addresses
-            for addr in ${addresses[@]}; do
-                printf "delete address $addr on $port\n"
-                ip addr del $addr dev $port || true
-                printf "add/replace address $addr to $br\n"
-                ip addr replace $addr dev $br
-            done
-
-            # transfer IPv4 routes
-            default_ipv4_routes=()
-            for route in ${ipv4_routes[@]}; do
-                r=$(echo $route | tr '#' ' ')
-                if echo $r | grep -q -w 'scope link'; then
-                    printf "add/replace IPv4 route $r to $br\n"
-                    ip -4 route replace $r dev $br
-                else
-                    default_ipv4_routes=(${default_ipv4_routes[@]} $route)
-                fi
-            done
-            for route in ${default_ipv4_routes[@]}; do
-                r=$(echo $route | tr '#' ' ')
-                printf "add/replace IPv4 route $r to $br\n"
-                ip -4 route replace $r dev $br
-            done
-
-            # transfer IPv6 routes
-            default_ipv6_routes=()
-            for route in ${ipv6_routes[@]}; do
-                r=$(echo $route | tr '#' ' ')
-                if echo $r | grep -q -w 'scope link'; then
-                    printf "add/replace IPv6 route $r to $br\n"
-                    ip -6 route replace $r dev $br
-                else
-                    default_ipv6_routes=(${default_ipv6_routes[@]} $route)
-                fi
-            done
-            for route in ${default_ipv6_routes[@]}; do
-                r=$(echo $route | tr '#' ' ')
-                printf "add/replace IPv6 route $r to $br\n"
-                ip -6 route replace $r dev $br
-            done
-        done
-    done
-}
-
-handle_underlay_bridges
-
 function gen_conn_str {
   if [[ -z "${OVN_DB_IPS}" ]]; then
     if [[ "$ENABLE_SSL" == "false" ]]; then
@@ -291,21 +134,19 @@ else
   /usr/share/ovn/scripts/ovn-ctl --ovn-controller-ssl-key=/var/run/tls/key --ovn-controller-ssl-cert=/var/run/tls/cert --ovn-controller-ssl-ca-cert=/var/run/tls/cacert restart_controller
 fi
 
-if [ $skip_wait_flows -eq 0 ]; then
-  # Wait ovn-controller finish init flow compute and update it to vswitchd,
-  # then update flow-restore-wait to indicate vswitchd to process flows
-  set +e
+# Wait ovn-controller finish init flow compute and update it to vswitchd,
+# then update flow-restore-wait to indicate vswitchd to process flows
+set +e
+flow_num=$(ovs-ofctl dump-flows br-int | wc -l)
+while [ $flow_num -le $FLOW_LIMIT ]
+do
+  echo "$flow_num flows now, waiting for ovs-vswitchd flow ready"
+  sleep 1
   flow_num=$(ovs-ofctl dump-flows br-int | wc -l)
-  while [ $flow_num -le $FLOW_LIMIT ]
-  do
-    echo "$flow_num flows now, waiting for ovs-vswitchd flow ready"
-    sleep 1
-    flow_num=$(ovs-ofctl dump-flows br-int | wc -l)
-  done
-  set -e
+done
+set -e
 
-  ovs-vsctl --no-wait set open_vswitch . other_config:flow-restore-wait="false"
-fi
+ovs-vsctl --no-wait set open_vswitch . other_config:flow-restore-wait="false"
 
 set +e
 for netns in /var/run/netns/*; do

diff --git a/go.mod b/go.mod
@@ -4,7 +4,7 @@ go 1.19
 
 require (
 	github.com/Mellanox/sriovnet v1.0.3
-	github.com/Wifx/gonetworkmanager v0.4.0
+	github.com/Wifx/gonetworkmanager v0.5.0
 	github.com/alauda/felix v3.6.6-0.20201207121355-187332daf314+incompatible
 	github.com/cenkalti/backoff/v4 v4.2.0
 	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08

diff --git a/go.sum b/go.sum
@@ -110,8 +110,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
-github.com/Wifx/gonetworkmanager v0.4.0 h1:iZ7o3z3YIqEEAa+bBOCwQkLLUQuxG02CdiE7NCe5y6A=
-github.com/Wifx/gonetworkmanager v0.4.0/go.mod h1:EdhHf2O00IZXfMv9LC6CS6SgTwcMTg/ZSDhGvch0cs8=
+github.com/Wifx/gonetworkmanager v0.5.0 h1:P209z0yj705bl5tmyHTlpXPSv3QzjPtIM4X0SyDAqWA=
+github.com/Wifx/gonetworkmanager v0.5.0/go.mod h1:EdhHf2O00IZXfMv9LC6CS6SgTwcMTg/ZSDhGvch0cs8=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/alauda/ovsdb v0.0.0-20210113100339-040cf3e76c28 h1:FW5M3SAwSGBdtTboeV5sI7kEY6zraApSZQxTUfZ7LQY=
 github.com/alauda/ovsdb v0.0.0-20210113100339-040cf3e76c28/go.mod h1:dXpg+IAC2yp2IZQlEVmnmEc1rqEmSZzgNfu6+ai38J4=

diff --git a/pkg/daemon/controller.go b/pkg/daemon/controller.go
@@ -343,6 +343,10 @@ func (c *Controller) recordProviderNetworkErr(providerNetwork string, errMsg str
 				break
 			}
 		}
+		if currentPod == nil {
+			klog.Warning("failed to get self pod")
+			return
+		}
 	} else {
 		if currentPod, err = c.config.KubeClient.CoreV1().Pods(c.localNamespace).Get(context.Background(), c.localPodName, metav1.GetOptions{}); err != nil {
 			klog.Errorf("failed to get pod %s, %v", c.localPodName, err)