fix iptables for services with external traffic policy set to Local (#…

…1773) * fix iptables for services with external traffic policy set to Local * do not retry e2e testing
kubeovn · Aug 8, 2022 · 277f6f6 · 277f6f6
1 parent 42812b9
commit 277f6f6
Show file tree

Hide file tree

Showing 3 changed files with 71 additions and 122 deletions.
diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml
@@ -140,17 +140,12 @@ jobs:
         id: go
 
       - name: Run E2E
-        uses: nick-invision/retry@v2
-        with:
-          timeout_minutes: 10
-          max_attempts: 2
-          shell: bash
-          command: |
-            go install github.com/onsi/ginkgo/ginkgo@latest
-            sudo kubectl cluster-info
-            sudo cp -r /root/.kube/ /home/runner/.kube/
-            sudo chmod -R 777 /home/runner/.kube/
-            make e2e
+        run: |
+          go install github.com/onsi/ginkgo/ginkgo@latest
+          sudo kubectl cluster-info
+          sudo cp -r /root/.kube/ /home/runner/.kube/
+          sudo chmod -R 777 /home/runner/.kube/
+          make e2e
 
       - name: Cleanup
         run: |
@@ -263,17 +258,12 @@ jobs:
         id: go
 
       - name: Run E2E
-        uses: nick-invision/retry@v2
-        with:
-          timeout_minutes: 10
-          max_attempts: 2
-          shell: bash
-          command: |
-            go install github.com/onsi/ginkgo/ginkgo@latest
-            sudo kubectl cluster-info
-            sudo cp -r /root/.kube/ /home/runner/.kube/
-            sudo chmod -R 777 /home/runner/.kube/
-            make e2e-vlan
+        run: |
+          go install github.com/onsi/ginkgo/ginkgo@latest
+          sudo kubectl cluster-info
+          sudo cp -r /root/.kube/ /home/runner/.kube/
+          sudo chmod -R 777 /home/runner/.kube/
+          make e2e-vlan
 
       - name: Cleanup
         run: |
@@ -465,17 +455,12 @@ jobs:
         id: go
 
       - name: Run E2E
-        uses: nick-invision/retry@v2
-        with:
-          timeout_minutes: 10
-          max_attempts: 2
-          shell: bash
-          command: |
-            go install github.com/onsi/ginkgo/ginkgo@latest
-            sudo kubectl cluster-info
-            sudo cp -r /root/.kube/ /home/runner/.kube/
-            sudo chmod -R 777 /home/runner/.kube/
-            make e2e
+        run: |
+          go install github.com/onsi/ginkgo/ginkgo@latest
+          sudo kubectl cluster-info
+          sudo cp -r /root/.kube/ /home/runner/.kube/
+          sudo chmod -R 777 /home/runner/.kube/
+          make e2e
 
       - name: Cleanup
         run: |
@@ -529,17 +514,12 @@ jobs:
         id: go
 
       - name: Run E2E
-        uses: nick-invision/retry@v2
-        with:
-          timeout_minutes: 10
-          max_attempts: 2
-          shell: bash
-          command: |
-            go install github.com/onsi/ginkgo/ginkgo@latest
-            sudo kubectl cluster-info
-            sudo cp -r /root/.kube/ /home/runner/.kube/
-            sudo chmod -R 777 /home/runner/.kube/
-            make e2e-ipv6
+        run: |
+          go install github.com/onsi/ginkgo/ginkgo@latest
+          sudo kubectl cluster-info
+          sudo cp -r /root/.kube/ /home/runner/.kube/
+          sudo chmod -R 777 /home/runner/.kube/
+          make e2e-ipv6
 
       - name: Cleanup
         run: |
@@ -593,17 +573,12 @@ jobs:
         id: go
 
       - name: Run E2E
-        uses: nick-invision/retry@v2
-        with:
-          timeout_minutes: 10
-          max_attempts: 2
-          shell: bash
-          command: |
-            go install github.com/onsi/ginkgo/ginkgo@latest
-            sudo kubectl cluster-info
-            sudo cp -r /root/.kube/ /home/runner/.kube/
-            sudo chmod -R 777 /home/runner/.kube/
-            make e2e-vlan-ipv6
+        run: |
+          go install github.com/onsi/ginkgo/ginkgo@latest
+          sudo kubectl cluster-info
+          sudo cp -r /root/.kube/ /home/runner/.kube/
+          sudo chmod -R 777 /home/runner/.kube/
+          make e2e-vlan-ipv6
 
       - name: Cleanup
         run: |
@@ -732,17 +707,12 @@ jobs:
         id: go
 
       - name: Run E2E
-        uses: nick-invision/retry@v2
-        with:
-          timeout_minutes: 10
-          max_attempts: 2
-          shell: bash
-          command: |
-            go install github.com/onsi/ginkgo/ginkgo@latest
-            sudo kubectl cluster-info
-            sudo cp -r /root/.kube/ /home/runner/.kube/
-            sudo chmod -R 777 /home/runner/.kube/
-            make e2e
+        run: |
+          go install github.com/onsi/ginkgo/ginkgo@latest
+          sudo kubectl cluster-info
+          sudo cp -r /root/.kube/ /home/runner/.kube/
+          sudo chmod -R 777 /home/runner/.kube/
+          make e2e
 
       - name: Cleanup
         run: |
@@ -871,17 +841,12 @@ jobs:
         id: go
 
       - name: Run E2E
-        uses: nick-invision/retry@v2
-        with:
-          timeout_minutes: 10
-          max_attempts: 2
-          shell: bash
-          command: |
-            go install github.com/onsi/ginkgo/ginkgo@latest
-            sudo kubectl cluster-info
-            sudo cp -r /root/.kube/ /home/runner/.kube/
-            sudo chmod -R 777 /home/runner/.kube/
-            make e2e
+        run: |
+          go install github.com/onsi/ginkgo/ginkgo@latest
+          sudo kubectl cluster-info
+          sudo cp -r /root/.kube/ /home/runner/.kube/
+          sudo chmod -R 777 /home/runner/.kube/
+          make e2e
 
       - name: Cleanup
         run: |
@@ -953,17 +918,12 @@ jobs:
         id: go
 
       - name: Run E2E
-        uses: nick-invision/retry@v2
-        with:
-          timeout_minutes: 10
-          max_attempts: 2
-          shell: bash
-          command: |
-            go install github.com/onsi/ginkgo/ginkgo@latest
-            sudo kubectl cluster-info
-            sudo cp -r /root/.kube/ /home/runner/.kube/
-            sudo chmod -R 777 /home/runner/.kube/
-            make e2e
+        run: |
+          go install github.com/onsi/ginkgo/ginkgo@latest
+          sudo kubectl cluster-info
+          sudo cp -r /root/.kube/ /home/runner/.kube/
+          sudo chmod -R 777 /home/runner/.kube/
+          make e2e
 
       - name: Cleanup
         run: |
@@ -1078,17 +1038,12 @@ jobs:
         id: go
 
       - name: Run E2E
-        uses: nick-invision/retry@v2
-        with:
-          timeout_minutes: 10
-          max_attempts: 2
-          shell: bash
-          command: |
-            go install github.com/onsi/ginkgo/ginkgo@latest
-            sudo kubectl cluster-info
-            sudo cp -r /root/.kube/ /home/runner/.kube/
-            sudo chmod -R 777 /home/runner/.kube/
-            make e2e
+        run: |
+          go install github.com/onsi/ginkgo/ginkgo@latest
+          sudo kubectl cluster-info
+          sudo cp -r /root/.kube/ /home/runner/.kube/
+          sudo chmod -R 777 /home/runner/.kube/
+          make e2e
 
       - name: Cleanup
         run: |

diff --git a/dist/images/uninstall.sh b/dist/images/uninstall.sh
@@ -9,8 +9,8 @@ iptables -t nat -D POSTROUTING -m mark --mark 0x80000/0x80000 -j MASQUERADE
 iptables -t nat -D POSTROUTING -p tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN
 iptables -t nat -D POSTROUTING -m set ! --match-set ovn40subnets src -m set ! --match-set ovn40other-node src -m set --match-set ovn40subnets-nat dst -j RETURN
 iptables -t nat -D POSTROUTING -m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE
-iptables -t nat -D KUBE-NODE-PORT -p tcp -m set --match-set KUBE-NODE-PORT-LOCAL-TCP dst -j MARK --set-xmark 0x80000/0x80000
-iptables -t nat -D KUBE-NODE-PORT -p udp -m set --match-set KUBE-NODE-PORT-LOCAL-UDP dst -j MARK --set-xmark 0x80000/0x80000
+iptables -t nat -D PREROUTING -p tcp -m addrtype --dst-type LOCAL -m set --match-set KUBE-NODE-PORT-LOCAL-TCP dst -j MARK --set-xmark 0x80000/0x80000
+iptables -t nat -D PREROUTING -p udp -m addrtype --dst-type LOCAL -m set --match-set KUBE-NODE-PORT-LOCAL-UDP dst -j MARK --set-xmark 0x80000/0x80000
 iptables -t mangle -D PREROUTING -i ovn0 -m set --match-set ovn40subnets src -m set --match-set ovn40services dst -j MARK --set-xmark 0x4000/0x4000
 iptables -t filter -D INPUT -m set --match-set ovn40subnets dst -j ACCEPT
 iptables -t filter -D INPUT -m set --match-set ovn40subnets src -j ACCEPT
@@ -38,8 +38,8 @@ ip6tables -t nat -D POSTROUTING -m mark --mark 0x80000/0x80000 -j MASQUERADE
 ip6tables -t nat -D POSTROUTING -p tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN
 ip6tables -t nat -D POSTROUTING -m set ! --match-set ovn60subnets src -m set ! --match-set ovn60other-node src -m set --match-set ovn60subnets-nat dst -j RETURN
 ip6tables -t nat -D POSTROUTING -m set --match-set ovn60subnets-nat src -m set ! --match-set ovn60subnets dst -j MASQUERADE
-ip6tables -t nat -D KUBE-NODE-PORT -p tcp -m set --match-set KUBE-6-NODE-PORT-LOCAL-TCP dst -j MARK --set-xmark 0x80000/0x80000
-ip6tables -t nat -D KUBE-NODE-PORT -p udp -m set --match-set KUBE-6-NODE-PORT-LOCAL-UDP dst -j MARK --set-xmark 0x80000/0x80000
+ip6tables -t nat -D PREROUTING -p tcp -m addrtype --dst-type LOCAL -m set --match-set KUBE-6-NODE-PORT-LOCAL-TCP dst -j MARK --set-xmark 0x80000/0x80000
+ip6tables -t nat -D PREROUTING -p udp -m addrtype --dst-type LOCAL -m set --match-set KUBE-6-NODE-PORT-LOCAL-UDP dst -j MARK --set-xmark 0x80000/0x80000
 ip6tables -t mangle -D PREROUTING -i ovn0 -m set --match-set ovn60subnets src -m set --match-set ovn60services dst -j MARK --set-xmark 0x4000/0x4000
 ip6tables -t filter -D INPUT -m set --match-set ovn60subnets dst -j ACCEPT
 ip6tables -t filter -D INPUT -m set --match-set ovn60subnets src -j ACCEPT

diff --git a/pkg/daemon/gateway_linux.go b/pkg/daemon/gateway_linux.go
@@ -421,27 +421,21 @@ func (c *Controller) setIptables() error {
 				util.IPTableRule{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(fmt.Sprintf(`! -s %s -m set ! --match-set %s src -m set --match-set %s dst -j MASQUERADE`, nodeIP, matchset, matchset))},
 			)
 
-			chainExists, err := c.iptables[protocol].ChainExists("nat", "KUBE-NODE-PORT")
-			if err != nil {
-				klog.Errorf("failed to check existence of chain KUBE-NODE-PORT in nat table: %v", err)
-				return err
-			}
-			if chainExists {
-				nodePortRules := make([]util.IPTableRule, 0, len(kubeProxyIpsets))
-				for protocol, ipset := range kubeProxyIpsets {
-					ipsetExists, err := ipsetExists(ipset)
-					if err != nil {
-						klog.Error("failed to check existence of ipset %s: %v", ipset, err)
-						return err
-					}
-					if !ipsetExists {
-						klog.Warningf("ipset %s does not exist", ipset)
-						continue
-					}
-					nodePortRules = append(nodePortRules, util.IPTableRule{Table: "nat", Chain: "KUBE-NODE-PORT", Rule: strings.Fields(fmt.Sprintf("-p %s -m set --match-set %s dst -j MARK --set-xmark 0x80000/0x80000", protocol, ipset))})
+			nodePortRules := make([]util.IPTableRule, 0, len(kubeProxyIpsets))
+			for protocol, ipset := range kubeProxyIpsets {
+				ipsetExists, err := ipsetExists(ipset)
+				if err != nil {
+					klog.Error("failed to check existence of ipset %s: %v", ipset, err)
+					return err
+				}
+				if !ipsetExists {
+					klog.Warningf("ipset %s does not exist", ipset)
+					continue
 				}
-				iptablesRules = append(nodePortRules, iptablesRules...)
+				rule := fmt.Sprintf("-p %s -m addrtype --dst-type LOCAL -m set --match-set %s dst -j MARK --set-xmark 0x80000/0x80000", protocol, ipset)
+				nodePortRules = append(nodePortRules, util.IPTableRule{Table: "nat", Chain: "PREROUTING", Rule: strings.Fields(rule)})
 			}
+			iptablesRules = append(nodePortRules, iptablesRules...)
 		}
 
 		// delete abandoned iptables rules