fix IPv6-related issues

kubeovn · Aug 18, 2021 · 2919288 · 2919288
1 parent aaf56e6
commit 2919288
Show file tree

Hide file tree

Showing 17 changed files with 215 additions and 99 deletions.
diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml
@@ -334,6 +334,20 @@ jobs:
           docker load --input image.tar
           sudo make kind-install-ipv6
 
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.16
+        id: go
+
+      - name: Run E2E
+        run: |
+          go get -u github.com/onsi/ginkgo/ginkgo
+          go get -u github.com/onsi/gomega/...
+          sudo kubectl cluster-info
+          sudo chmod 666 /home/runner/.kube/config
+          make e2e-ipv6
+
   ipv6-vlan-e2e-single-nic:
     needs: build
     name: ipv6-vlan-e2e-single-nic

diff --git a/.gitignore b/.gitignore
@@ -1,18 +1,10 @@
 .idea/*
 .DS_Store
-vagrant/.vagrant/*
-vagrant/kubernetes-setup/*.retry
-vagrant/kubernetes-setup/join-command
 dist/images/kube-ovn
-dist/images/kube-ovn-controller
-dist/images/kube-ovn-daemon
-dist/images/kube-ovn-gateway
-dist/images/kube-ovn-webhook
-dist/images/kube-ovn-pinger
-dist/images/kube-ovn-speaker
-dist/images/kube-ovn-monitor
 dist/images/kube-ovn-cmd
 kube-ovn.yaml
 kube-ovn-crd.yaml
 ovn.yaml
 kind.yaml
+install-vlan.sh
+test/**/*.test
diff --git a/Makefile b/Makefile
@@ -120,7 +120,7 @@ kind-install-vlan:
 	$(eval SUBNET = $(shell docker network inspect kind -f "{{(index .IPAM.Config 0).Subnet}}"))
 	$(eval GATEWAY = $(shell docker network inspect kind -f "{{(index .IPAM.Config 0).Gateway}}"))
 	$(eval EXCLUDE_IPS = $(shell docker network inspect kind -f '{{range .Containers}},{{index (split .IPv4Address "/") 0}}{{end}}' | sed 's/^,//'))
-	sed -e 's@^[[:space:]]*POD_CIDR=.*@POD_CIDR="$(SUBNET)"@' \
+	@sed -e 's@^[[:space:]]*POD_CIDR=.*@POD_CIDR="$(SUBNET)"@' \
 		-e 's@^[[:space:]]*POD_GATEWAY=.*@POD_GATEWAY="$(GATEWAY)"@' \
 		-e 's@^[[:space:]]*EXCLUDE_IPS=.*@EXCLUDE_IPS="$(EXCLUDE_IPS)"@' \
 		-e 's@^VLAN_ID=.*@VLAN_ID="0"@' \
@@ -148,12 +148,12 @@ kind-install-ipv6-vlan:
 	$(eval SUBNET = $(shell docker network inspect kind -f "{{(index .IPAM.Config 1).Subnet}}"))
 	$(eval EXCLUDE_IPS = $(shell docker network inspect kind -f '{{range .Containers}},{{index (split .IPv6Address "/") 0}}{{end}}' | sed 's/^,//'))
 	$(eval GATEWAY = $(shell docker exec kube-ovn-worker ip -6 route show default | awk '{print $$3}'))
-	sed -e 's@^[[:space:]]*POD_CIDR=.*@POD_CIDR="$(SUBNET)"@' \
+	@sed -e 's@^[[:space:]]*POD_CIDR=.*@POD_CIDR="$(SUBNET)"@' \
 		-e 's@^[[:space:]]*POD_GATEWAY=.*@POD_GATEWAY="$(GATEWAY)"@' \
 		-e 's@^[[:space:]]*EXCLUDE_IPS=.*@EXCLUDE_IPS="$(EXCLUDE_IPS)"@' \
 		-e 's@^VLAN_ID=.*@VLAN_ID="0"@' \
 		dist/images/install.sh > install-vlan.sh
-	chmod +x install-vlan.sh
+	@chmod +x install-vlan.sh
 	kind load docker-image --name kube-ovn $(REGISTRY)/kube-ovn:$(RELEASE_TAG)
 	kubectl taint node kube-ovn-control-plane node-role.kubernetes.io/master:NoSchedule-
 	ENABLE_SSL=true IPv6=true ENABLE_VLAN=true VLAN_NIC=eth0 ./install-vlan.sh
@@ -200,27 +200,31 @@ ut:
 .PHONY: e2e
 e2e:
 	$(eval NETWORK_BRIDGE = $(shell docker inspect -f '{{json .NetworkSettings.Networks.bridge}}' kube-ovn-control-plane))
-	if [ '$(NETWORK_BRIDGE)' = 'null' ]; then \
+	@if [ '$(NETWORK_BRIDGE)' = 'null' ]; then \
 		kind get nodes --name kube-ovn | while read node; do \
 		docker network connect bridge $$node; \
 		done; \
 	fi
 
-	printf "package e2e\n\nvar nodeNetworks = map[string]string{\n" > test/e2e/network.go
-	kind get nodes --name kube-ovn | while read node; do \
-		printf "    \`$$node\`: \`" >> test/e2e/network.go; \
+	@printf "package e2e\n\nvar nodeNetworks = map[string]string{\n" > test/e2e/network.go
+	@kind get nodes --name kube-ovn | while read node; do \
+		printf "\t\`$$node\`: \`" >> test/e2e/network.go; \
 		docker inspect -f '{{json .NetworkSettings.Networks.bridge}}' $$node >> test/e2e/network.go; \
 		printf "\`,\n" >> test/e2e/network.go; \
 	done
-	echo "}" >> test/e2e/network.go
+	@echo "}" >> test/e2e/network.go
 
-	docker pull kubeovn/pause:3.2
+	@if [ ! -n "$$(docker images -q kubeovn/pause:3.2 2>/dev/null)" ]; then docker pull kubeovn/pause:3.2; fi
 	kind load docker-image --name kube-ovn kubeovn/pause:3.2
 	ginkgo -mod=mod -progress -reportPassed --slowSpecThreshold=60 test/e2e
 
+.PHONY: e2e-ipv6
+e2e-ipv6:
+	@IPV6=true $(MAKE) e2e
+
 .PHONY: e2e-vlan-single-nic
 e2e-vlan-single-nic:
-	printf "package node\n\nvar networkJSON = []byte(\`" > test/e2e-vlan-single-nic/node/network.go
-	docker inspect -f '{{json .NetworkSettings.Networks.kind}}' kube-ovn-control-plane >> test/e2e-vlan-single-nic/node/network.go
-	echo "\`)" >> test/e2e-vlan-single-nic/node/network.go
+	@printf "package node\n\nvar networkJSON = []byte(\`" > test/e2e-vlan-single-nic/node/network.go
+	@docker inspect -f '{{json .NetworkSettings.Networks.kind}}' kube-ovn-control-plane >> test/e2e-vlan-single-nic/node/network.go
+	@echo "\`)" >> test/e2e-vlan-single-nic/node/network.go
 	ginkgo -mod=mod -progress -reportPassed --slowSpecThreshold=60 test/e2e-vlan-single-nic
diff --git a/dist/images/Dockerfile b/dist/images/Dockerfile
@@ -46,7 +46,7 @@ ARG RPM_ARCH
 
 RUN apt update && apt upgrade -y && apt install ca-certificates python3 hostname libunwind8 netbase \
         ethtool iproute2 ncat libunbound-dev procps libatomic1 kmod \
-        tcpdump ipset curl uuid-runtime openssl inetutils-ping arping \
+        tcpdump ipset curl uuid-runtime openssl inetutils-ping arping ndisc6 \
         logrotate -y --no-install-recommends && \
         rm -rf /var/lib/apt/lists/*
 

diff --git a/dist/images/install-pre-1.16.sh b/dist/images/install-pre-1.16.sh
@@ -2090,7 +2090,12 @@ trace(){
       nicName="eth0"
     fi
 
-    output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" arping -c1 -i "$nicName" "$gateway")
+    if [[ "$gateway" =~ .*:.* ]]; then
+      output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ndisc6 -q "$gateway" "$nicName")
+    else
+      output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" arping -c3 -C1 -i1 -I "$nicName" "$gateway")
+    fi
+
     if [ $? -ne 0 ]; then
       echo "failed to run 'arping -c1 -i $nicName $gateway' in Pod netns"
       exit 1
@@ -2113,14 +2118,23 @@ trace(){
 
   type="$3"
 
+  af="4"
+  nw="nw"
+  proto=""
+  if [[ "$podIP" =~ .*:.* ]]; then
+    af="6"
+    nw="ipv6"
+    proto="6"
+  fi
+
   case $type in
     icmp)
       set -x
-      kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && icmp && eth.src == $mac && ip4.src == $podIP && eth.dst == $gwMac && ip4.dst == $dst"
+      kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && icmp && eth.src == $mac && ip$af.src == $podIP && eth.dst == $gwMac && ip$af.dst == $dst"
       ;;
     tcp|udp)
       set -x
-      kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && eth.src == $mac && ip4.src == $podIP && eth.dst == $gwMac && ip4.dst == $dst && $type.src == 10000 && $type.dst == $4"
+      kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && eth.src == $mac && ip$af.src == $podIP && eth.dst == $gwMac && ip$af.dst == $dst && $type.src == 10000 && $type.dst == $4"
       ;;
     *)
       echo "type $type not supported"
@@ -2137,19 +2151,19 @@ trace(){
 
   ovsPod=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep " $nodeName " | grep ovs-ovn | awk '{print $1}')
   if [ -z "$ovsPod" ]; then
-      echo "ovs pod  doesn't exist on node $nodeName"
-      exit 1
+    echo "ovs pod doesn't exist on node $nodeName"
+    exit 1
   fi
 
   inPort=$(kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-vsctl --format=csv --data=bare --no-heading --columns=ofport find interface external_id:iface-id="$podName"."$namespace")
-    case $type in
+  case $type in
     icmp)
       set -x
-      kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int in_port="$inPort",icmp,nw_src="$podIP",nw_dst="$dst",dl_src="$mac",dl_dst="$gwMac"
+      kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,icmp$proto,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$gwMac"
       ;;
     tcp|udp)
       set -x
-      kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int in_port="$inPort","$type",nw_src="$podIP",nw_dst="$dst",dl_src="$mac",dl_dst="$gwMac","$type"_src=1000,"$type"_dst="$4"
+      kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,$type$proto,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$gwMac,${type}_src=1000,${type}_dst=$4"
       ;;
     *)
       echo "type $type not supported"
@@ -2165,8 +2179,8 @@ xxctl(){
   kubectl get no "$nodeName" > /dev/null
   ovsPod=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep " $nodeName " | grep ovs-ovn | awk '{print $1}')
   if [ -z "$ovsPod" ]; then
-      echo "ovs pod  doesn't exist on node $nodeName"
-      exit 1
+    echo "ovs pod  doesn't exist on node $nodeName"
+    exit 1
   fi
   kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-$subcommand "$@"
 }

diff --git a/dist/images/install.sh b/dist/images/install.sh
@@ -2312,7 +2312,12 @@ trace(){
       nicName="eth0"
     fi
 
-    output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" arping -c1 -i "$nicName" "$gateway")
+    if [[ "$gateway" =~ .*:.* ]]; then
+      output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ndisc6 -q "$gateway" "$nicName")
+    else
+      output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" arping -c3 -C1 -i1 -I "$nicName" "$gateway")
+    fi
+
     if [ $? -ne 0 ]; then
       echo "failed to run 'arping -c1 -i $nicName $gateway' in Pod netns"
       exit 1
@@ -2335,14 +2340,23 @@ trace(){
 
   type="$3"
 
+  af="4"
+  nw="nw"
+  proto=""
+  if [[ "$podIP" =~ .*:.* ]]; then
+    af="6"
+    nw="ipv6"
+    proto="6"
+  fi
+
   case $type in
     icmp)
       set -x
-      kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && icmp && eth.src == $mac && ip4.src == $podIP && eth.dst == $gwMac && ip4.dst == $dst"
+      kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && icmp && eth.src == $mac && ip$af.src == $podIP && eth.dst == $gwMac && ip$af.dst == $dst"
       ;;
     tcp|udp)
       set -x
-      kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && eth.src == $mac && ip4.src == $podIP && eth.dst == $gwMac && ip4.dst == $dst && $type.src == 10000 && $type.dst == $4"
+      kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && eth.src == $mac && ip$af.src == $podIP && eth.dst == $gwMac && ip$af.dst == $dst && $type.src == 10000 && $type.dst == $4"
       ;;
     *)
       echo "type $type not supported"
@@ -2359,19 +2373,19 @@ trace(){
 
   ovsPod=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep " $nodeName " | grep ovs-ovn | awk '{print $1}')
   if [ -z "$ovsPod" ]; then
-      echo "ovs pod  doesn't exist on node $nodeName"
-      exit 1
+    echo "ovs pod doesn't exist on node $nodeName"
+    exit 1
   fi
 
   inPort=$(kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-vsctl --format=csv --data=bare --no-heading --columns=ofport find interface external_id:iface-id="$podName"."$namespace")
-    case $type in
+  case $type in
     icmp)
       set -x
-      kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int in_port="$inPort",icmp,nw_src="$podIP",nw_dst="$dst",dl_src="$mac",dl_dst="$gwMac"
+      kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,icmp$proto,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$gwMac"
       ;;
     tcp|udp)
       set -x
-      kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int in_port="$inPort","$type",nw_src="$podIP",nw_dst="$dst",dl_src="$mac",dl_dst="$gwMac","$type"_src=1000,"$type"_dst="$4"
+      kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,$type$proto,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$gwMac,${type}_src=1000,${type}_dst=$4"
       ;;
     *)
       echo "type $type not supported"
@@ -2387,8 +2401,8 @@ xxctl(){
   kubectl get no "$nodeName" > /dev/null
   ovsPod=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep " $nodeName " | grep ovs-ovn | awk '{print $1}')
   if [ -z "$ovsPod" ]; then
-      echo "ovs pod  doesn't exist on node $nodeName"
-      exit 1
+    echo "ovs pod  doesn't exist on node $nodeName"
+    exit 1
   fi
   kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-$subcommand "$@"
 }

diff --git a/dist/images/kubectl-ko b/dist/images/kubectl-ko
@@ -52,7 +52,7 @@ tcpdump(){
       exit 1
     fi
     podNicType=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/pod_nic_type})
-        podNetNs=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading get interface "$nicName" external-ids:pod_netns | tr -d '\r' | sed -e 's/^"//' -e 's/"$//')
+    podNetNs=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading get interface "$nicName" external-ids:pod_netns | tr -d '\r' | sed -e 's/^"//' -e 's/"$//')
     set -x
     if [ "$podNicType" = "internal-port" ]; then
       kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" tcpdump -nn -i "$nicName" "$@"
@@ -111,7 +111,12 @@ trace(){
       nicName="eth0"
     fi
 
-    output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" arping -c1 -i "$nicName" "$gateway")
+    if [[ "$gateway" =~ .*:.* ]]; then
+      output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ndisc6 -q "$gateway" "$nicName")
+    else
+      output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" arping -c3 -C1 -i1 -I "$nicName" "$gateway")
+    fi
+
     if [ $? -ne 0 ]; then
       echo "failed to run 'arping -c1 -i $nicName $gateway' in Pod netns"
       exit 1
@@ -134,14 +139,23 @@ trace(){
 
   type="$3"
 
+  af="4"
+  nw="nw"
+  proto=""
+  if [[ "$podIP" =~ .*:.* ]]; then
+    af="6"
+    nw="ipv6"
+    proto="6"
+  fi
+
   case $type in
     icmp)
       set -x
-      kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && icmp && eth.src == $mac && ip4.src == $podIP && eth.dst == $gwMac && ip4.dst == $dst"
+      kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && icmp && eth.src == $mac && ip$af.src == $podIP && eth.dst == $gwMac && ip$af.dst == $dst"
       ;;
     tcp|udp)
       set -x
-      kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && eth.src == $mac && ip4.src == $podIP && eth.dst == $gwMac && ip4.dst == $dst && $type.src == 10000 && $type.dst == $4"
+      kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && eth.src == $mac && ip$af.src == $podIP && eth.dst == $gwMac && ip$af.dst == $dst && $type.src == 10000 && $type.dst == $4"
       ;;
     *)
       echo "type $type not supported"
@@ -158,19 +172,19 @@ trace(){
 
   ovsPod=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep " $nodeName " | grep ovs-ovn | awk '{print $1}')
   if [ -z "$ovsPod" ]; then
-      echo "ovs pod  doesn't exist on node $nodeName"
-      exit 1
+    echo "ovs pod doesn't exist on node $nodeName"
+    exit 1
   fi
 
   inPort=$(kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-vsctl --format=csv --data=bare --no-heading --columns=ofport find interface external_id:iface-id="$podName"."$namespace")
-    case $type in
+  case $type in
     icmp)
       set -x
-      kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int in_port="$inPort",icmp,nw_src="$podIP",nw_dst="$dst",dl_src="$mac",dl_dst="$gwMac"
+      kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,icmp$proto,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$gwMac"
       ;;
     tcp|udp)
       set -x
-      kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int in_port="$inPort","$type",nw_src="$podIP",nw_dst="$dst",dl_src="$mac",dl_dst="$gwMac","$type"_src=1000,"$type"_dst="$4"
+      kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,$type$proto,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$gwMac,${type}_src=1000,${type}_dst=$4"
       ;;
     *)
       echo "type $type not supported"
@@ -186,8 +200,8 @@ xxctl(){
   kubectl get no "$nodeName" > /dev/null
   ovsPod=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep " $nodeName " | grep ovs-ovn | awk '{print $1}')
   if [ -z "$ovsPod" ]; then
-      echo "ovs pod  doesn't exist on node $nodeName"
-      exit 1
+    echo "ovs pod  doesn't exist on node $nodeName"
+    exit 1
   fi
   kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-$subcommand "$@"
 }
@@ -213,6 +227,8 @@ diagnose(){
   kubectl get crd vpc-nat-gateways.kubeovn.io
   kubectl get crd subnets.kubeovn.io
   kubectl get crd ips.kubeovn.io
+  kubectl get crd vlans.kubeovn.io
+  kubectl get crd provider-networks.kubeovn.io
   kubectl get svc kube-dns -n kube-system
   kubectl get svc kubernetes -n default
   kubectl get sa -n kube-system ovn

diff --git a/pkg/controller/subnet.go b/pkg/controller/subnet.go
@@ -944,7 +944,7 @@ func (c *Controller) reconcileGateway(subnet *kubeovnv1.Subnet) error {
 				return fmt.Errorf("failed to add ecmp static route, no gateway node exists")
 			}
 
-			nodeIPs := make([]string, len(strings.Split(subnet.Spec.GatewayNode, ",")))
+			nodeIPs := make([]string, 0, len(strings.Split(subnet.Spec.GatewayNode, ",")))
 			for _, gw := range strings.Split(subnet.Spec.GatewayNode, ",") {
 				gw = strings.TrimSpace(gw)
 				node, err := c.nodesLister.Get(gw)