fix iptables for direct routing (#1578)

kubeovn · Jul 7, 2022 · 874785b · 874785b
1 parent f3886af
commit 874785b
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 43 deletions.
diff --git a/dist/images/cleanup.sh b/dist/images/cleanup.sh
@@ -43,9 +43,7 @@ while :; do
 done
 
 for pod in $(kubectl get pod -n kube-system -l app=ovs -o 'jsonpath={.items[?(@.status.phase=="Running")].metadata.name}'); do
-  node=$(kubectl get pod -n kube-system $pod -o 'jsonpath={.spec.nodeName}')
-  nodeIPs=$(kubectl get node $node -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}' | sed 's/ /,/')
-  kubectl exec -n kube-system "$pod" -- bash /kube-ovn/uninstall.sh "$nodeIPs"
+  kubectl exec -n kube-system "$pod" -- bash /kube-ovn/uninstall.sh
 done
 
 kubectl delete --ignore-not-found svc ovn-nb ovn-sb ovn-northd -n kube-system

diff --git a/dist/images/uninstall.sh b/dist/images/uninstall.sh
@@ -2,29 +2,13 @@
 /usr/share/openvswitch/scripts/ovs-ctl stop
 ovs-dpctl del-dp ovs-system
 
-nodeIPv4=""
-nodeIPv6=""
-if [ -n "$1" ]; then
-    if [[ "$1" =~ .*,.* ]]; then
-        nodeIPv4=${1%%,*}
-        nodeIPv6=${1##*,}
-        if [[ "$nodeIPv4" =~ .*:.* ]]; then
-            nodeIPv4=${1##*,}
-            nodeIPv6=${1%%,*}
-        fi
-    else
-        if [[ "$1" =~ .*:.* ]]; then
-            nodeIPv6=$1
-        else
-            nodeIPv4=$1
-        fi
-    fi
-fi
-
+iptables -t nat -D POSTROUTING -m mark --mark 0x4000/0x4000 -j MASQUERADE
 iptables -t nat -D POSTROUTING -m mark --mark 0x80000/0x80000 -j RETURN
+iptables -t nat -D POSTROUTING -p tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN
 iptables -t nat -D POSTROUTING -m set ! --match-set ovn40subnets src -m set ! --match-set ovn40other-node src -m set --match-set ovn40subnets-nat dst -j RETURN
-ip6tables -t nat -D KUBE-NODE-PORT -p tcp -m set --match-set KUBE-NODE-PORT-LOCAL-TCP dst -j MARK --set-xmark 0x80000/0x80000
-ip6tables -t nat -D KUBE-NODE-PORT -p udp -m set --match-set KUBE-NODE-PORT-LOCAL-UDP dst -j MARK --set-xmark 0x80000/0x80000
+iptables -t nat -D POSTROUTING -m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE
+iptables -t nat -D KUBE-NODE-PORT -p tcp -m set --match-set KUBE-NODE-PORT-LOCAL-TCP dst -j MARK --set-xmark 0x80000/0x80000
+iptables -t nat -D KUBE-NODE-PORT -p udp -m set --match-set KUBE-NODE-PORT-LOCAL-UDP dst -j MARK --set-xmark 0x80000/0x80000
 iptables -t mangle -D PREROUTING -i ovn0 -m set --match-set ovn40subnets src -m set --match-set ovn40services dst -j MARK --set-xmark 0x4000/0x4000
 iptables -t filter -D INPUT -m set --match-set ovn40subnets dst -j ACCEPT
 iptables -t filter -D INPUT -m set --match-set ovn40subnets src -j ACCEPT
@@ -36,11 +20,6 @@ iptables -t filter -D FORWARD -m set --match-set ovn40services dst -j ACCEPT
 iptables -t filter -D FORWARD -m set --match-set ovn40services src -j ACCEPT
 iptables -t filter -D OUTPUT -p udp -m udp --dport 6081 -j MARK --set-xmark 0x0
 
-if [ -n "$nodeIPv4" ]; then
-    iptables -t nat -D POSTROUTING ! -s "$nodeIPv4" -m mark --mark 0x4000/0x4000 -j MASQUERADE
-    iptables -t nat -D POSTROUTING ! -s "$nodeIPv4" -m set ! --match-set ovn40subnets src -m set --match-set ovn40subnets dst -j MASQUERADE
-fi
-
 sleep 1
 
 ipset destroy ovn40subnets-nat
@@ -49,8 +28,11 @@ ipset destroy ovn40local-pod-ip-nat
 ipset destroy ovn40other-node
 ipset destroy ovn40services
 
+ip6tables -t nat -D POSTROUTING -m mark --mark 0x4000/0x4000 -j MASQUERADE
 ip6tables -t nat -D POSTROUTING -m mark --mark 0x80000/0x80000 -j RETURN
+ip6tables -t nat -D POSTROUTING -p tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN
 ip6tables -t nat -D POSTROUTING -m set ! --match-set ovn60subnets src -m set ! --match-set ovn60other-node src -m set --match-set ovn60subnets-nat dst -j RETURN
+ip6tables -t nat -D POSTROUTING -m set --match-set ovn60subnets-nat src -m set ! --match-set ovn60subnets dst -j MASQUERADE
 ip6tables -t nat -D KUBE-NODE-PORT -p tcp -m set --match-set KUBE-6-NODE-PORT-LOCAL-TCP dst -j MARK --set-xmark 0x80000/0x80000
 ip6tables -t nat -D KUBE-NODE-PORT -p udp -m set --match-set KUBE-6-NODE-PORT-LOCAL-UDP dst -j MARK --set-xmark 0x80000/0x80000
 ip6tables -t mangle -D PREROUTING -i ovn0 -m set --match-set ovn60subnets src -m set --match-set ovn60services dst -j MARK --set-xmark 0x4000/0x4000
@@ -64,11 +46,6 @@ ip6tables -t filter -D FORWARD -m set --match-set ovn60services dst -j ACCEPT
 ip6tables -t filter -D FORWARD -m set --match-set ovn60services src -j ACCEPT
 ip6tables -t filter -D OUTPUT -p udp -m udp --dport 6081 -j MARK --set-xmark 0x0
 
-if [ -n "$nodeIPv6" ]; then
-    ip6tables -t nat -D POSTROUTING ! -s "$nodeIPv6" -m mark --mark 0x4000/0x4000 -j MASQUERADE
-    ip6tables -t nat -D POSTROUTING ! -s "$nodeIPv6" -m set ! --match-set ovn60subnets src -m set --match-set ovn60subnets dst -j MASQUERADE
-fi
-
 sleep 1
 
 ipset destroy ovn6subnets-nat

diff --git a/pkg/daemon/gateway_linux.go b/pkg/daemon/gateway_linux.go
@@ -310,8 +310,12 @@ func (c *Controller) setIptables() error {
 		}
 
 		v4Rules = []util.IPTableRule{
+			// nat packets marked by kube-proxy or kube-ovn
+			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x4000/0x4000 -j MASQUERADE`)},
 			// do not nat node port service traffic with external traffic policy set to local
 			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j RETURN`)},
+			// do not nat reply packets in direct routing
+			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-p tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN`)},
 			// do not nat route traffic
 			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set ! --match-set ovn40subnets src -m set ! --match-set ovn40other-node src -m set --match-set ovn40subnets-nat dst -j RETURN`)},
 			// nat outgoing
@@ -332,8 +336,12 @@ func (c *Controller) setIptables() error {
 			{Table: "filter", Chain: "OUTPUT", Rule: strings.Fields(`-p udp -m udp --dport 6081 -j MARK --set-xmark 0x0`)},
 		}
 		v6Rules = []util.IPTableRule{
+			// nat packets marked by kube-proxy or kube-ovn
+			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x4000/0x4000 -j MASQUERADE`)},
 			// do not nat node port service traffic with external traffic policy set to local
 			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j RETURN`)},
+			// do not nat reply packets in direct routing
+			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-p tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN`)},
 			// do not nat route traffic
 			{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set ! --match-set ovn60subnets src -m set ! --match-set ovn60other-node src -m set --match-set ovn60subnets-nat dst -j RETURN`)},
 			// nat outgoing
@@ -390,14 +398,10 @@ func (c *Controller) setIptables() error {
 		if nodeIP := nodeIPs[protocol]; nodeIP != "" {
 			abandonedRules = append(abandonedRules,
 				util.IPTableRule{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(fmt.Sprintf(`! -s %s -m set --match-set %s dst -j MASQUERADE`, nodeIP, matchset))},
+				util.IPTableRule{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(fmt.Sprintf(`! -s %s -m mark --mark 0x4000/0x4000 -j MASQUERADE`, nodeIP))},
+				util.IPTableRule{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(fmt.Sprintf(`! -s %s -m set ! --match-set %s src -m set --match-set %s dst -j MASQUERADE`, nodeIP, matchset, matchset))},
 			)
 
-			rules := make([]util.IPTableRule, len(iptablesRules)+2)
-			copy(rules[1:4], iptablesRules[:3])
-			rules[0] = util.IPTableRule{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(fmt.Sprintf(`! -s %s -m mark --mark 0x4000/0x4000 -j MASQUERADE`, nodeIP))}
-			rules[4] = util.IPTableRule{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(fmt.Sprintf(`! -s %s -m set ! --match-set %s src -m set --match-set %s dst -j MASQUERADE`, nodeIP, matchset, matchset))}
-			copy(rules[5:], iptablesRules[3:])
-
 			chainExists, err := c.iptables[protocol].ChainExists("nat", "KUBE-NODE-PORT")
 			if err != nil {
 				klog.Errorf("failed to check existence of chain KUBE-NODE-PORT in nat table: %v", err)
@@ -417,10 +421,8 @@ func (c *Controller) setIptables() error {
 					}
 					nodePortRules = append(nodePortRules, util.IPTableRule{Table: "nat", Chain: "KUBE-NODE-PORT", Rule: strings.Fields(fmt.Sprintf("-p %s -m set --match-set %s dst -j MARK --set-xmark 0x80000/0x80000", protocol, ipset))})
 				}
-				rules = append(nodePortRules, rules...)
+				iptablesRules = append(nodePortRules, iptablesRules...)
 			}
-
-			iptablesRules = rules
 		}
 
 		// delete abandoned iptables rules