add subnet.spec.u2oInterconnectionIP (#2921)

* add subnet.spec.u2oInterconnectionIP

* fix log

charts/templates/kube-ovn-crd.yaml

@@ -1749,6 +1749,8 @@ spec:
                             type: string
                 u2oInterconnection:
                   type: boolean
+                u2oInterconnectionIP:
+                  type: string
                 enableLb:
                   type: boolean
                 enableEcmp:

dist/images/install.sh

@@ -2287,6 +2287,8 @@ spec:
                             type: string
                 u2oInterconnection:
                   type: boolean
+                u2oInterconnectionIP:
+                  type: string
                 enableLb:
                   type: boolean
                 enableEcmp:

pkg/apis/kubeovn/v1/types.go

@@ -161,9 +161,10 @@ type SubnetSpec struct {
 
 	NatOutgoingPolicyRules []NatOutgoingPolicyRule `json:"natOutgoingPolicyRules,omitempty"`
 
-	U2OInterconnection bool  `json:"u2oInterconnection,omitempty"`
-	EnableLb           *bool `json:"enableLb,omitempty"`
-	EnableEcmp         bool  `json:"enableEcmp,omitempty"`
+	U2OInterconnectionIP string `json:"u2oInterconnectionIP,omitempty"`
+	U2OInterconnection   bool   `json:"u2oInterconnection,omitempty"`
+	EnableLb             *bool  `json:"enableLb,omitempty"`
+	EnableEcmp           bool   `json:"enableEcmp,omitempty"`
 
 	RouteTable string `json:"routeTable,omitempty"`
 }

pkg/controller/subnet.go

@@ -103,7 +103,9 @@ func (c *Controller) enqueueUpdateSubnet(old, new interface{}) {
 		oldSubnet.Spec.U2OInterconnection != newSubnet.Spec.U2OInterconnection ||
 		oldSubnet.Spec.RouteTable != newSubnet.Spec.RouteTable ||
 		oldSubnet.Spec.Vpc != newSubnet.Spec.Vpc ||
-		!reflect.DeepEqual(oldSubnet.Spec.NatOutgoingPolicyRules, newSubnet.Spec.NatOutgoingPolicyRules) {
+		!reflect.DeepEqual(oldSubnet.Spec.NatOutgoingPolicyRules, newSubnet.Spec.NatOutgoingPolicyRules) ||
+		(newSubnet.Spec.U2OInterconnection && newSubnet.Spec.U2OInterconnectionIP != "" &&
+			oldSubnet.Spec.U2OInterconnectionIP != newSubnet.Spec.U2OInterconnectionIP) {
 		klog.V(3).Infof("enqueue update subnet %s", key)
 
 		if oldSubnet.Spec.GatewayType != newSubnet.Spec.GatewayType {
@@ -311,6 +313,11 @@ func formatSubnet(subnet *kubeovnv1.Subnet, c *Controller) (*kubeovnv1.Subnet, e
 		subnet.Spec.EnableLb = nil
 	}
 
+	if subnet.Spec.U2OInterconnectionIP != "" && !subnet.Spec.U2OInterconnection {
+		subnet.Spec.U2OInterconnectionIP = ""
+		changed = true
+	}
+
 	klog.Infof("format subnet %v, changed %v", subnet.Name, changed)
 	if changed {
 		subnet, err = c.config.KubeOvnClient.KubeovnV1().Subnets().Update(context.Background(), subnet, metav1.UpdateOptions{})
@@ -1803,14 +1810,29 @@ func (c *Controller) reconcileU2OInterconnectionIP(subnet *kubeovnv1.Subnet) err
 
 	needCalcIP := false
 	if subnet.Spec.U2OInterconnection {
-		if subnet.Status.U2OInterconnectionIP == "" {
-			u2oInterconnName := fmt.Sprintf(util.U2OInterconnName, subnet.Spec.Vpc, subnet.Name)
-			u2oInterconnLrpName := fmt.Sprintf("%s-%s", subnet.Spec.Vpc, subnet.Name)
-			v4ip, v6ip, _, err := c.acquireIpAddress(subnet.Name, u2oInterconnName, u2oInterconnLrpName)
+		u2oInterconnName := fmt.Sprintf(util.U2OInterconnName, subnet.Spec.Vpc, subnet.Name)
+		u2oInterconnLrpName := fmt.Sprintf("%s-%s", subnet.Spec.Vpc, subnet.Name)
+		var v4ip, v6ip string
+		var err error
+		if subnet.Spec.U2OInterconnectionIP == "" && subnet.Status.U2OInterconnectionIP == "" {
+			v4ip, v6ip, _, err = c.acquireIpAddress(subnet.Name, u2oInterconnName, u2oInterconnLrpName)
 			if err != nil {
 				klog.Errorf("failed to acquire underlay to overlay interconnection ip address for subnet %s, %v", subnet.Name, err)
 				return err
 			}
+		} else if subnet.Spec.U2OInterconnectionIP != "" && subnet.Status.U2OInterconnectionIP != subnet.Spec.U2OInterconnectionIP {
+			if subnet.Status.U2OInterconnectionIP != "" {
+				c.ipam.ReleaseAddressByPod(u2oInterconnName)
+			}
+
+			v4ip, v6ip, _, err = c.acquireStaticIpAddress(subnet.Name, u2oInterconnName, u2oInterconnLrpName, subnet.Spec.U2OInterconnectionIP)
+			if err != nil {
+				klog.Errorf("failed to acquire static underlay to overlay interconnection ip address for subnet %s, %v", subnet.Name, err)
+				return err
+			}
+		}
+
+		if v4ip != "" || v6ip != "" {
 			switch subnet.Spec.Protocol {
 			case kubeovnv1.ProtocolIPv4:
 				subnet.Status.U2OInterconnectionIP = v4ip

pkg/util/validator.go

@@ -115,6 +115,15 @@ func ValidateSubnet(subnet kubeovnv1.Subnet) error {
 			return err
 		}
 	}
+
+	if subnet.Spec.U2OInterconnectionIP != "" {
+		if !CIDRContainIP(subnet.Spec.CIDRBlock, subnet.Spec.U2OInterconnectionIP) {
+			return fmt.Errorf("u2oInterconnectionIP %s is not in subnet %s cidr %s",
+				subnet.Spec.U2OInterconnectionIP,
+				subnet.Name, subnet.Spec.CIDRBlock)
+		}
+	}
+
 	return nil
 }
 

test/e2e/kube-ovn/underlay/underlay.go

@@ -279,19 +279,29 @@ var _ = framework.SerialDescribe("[group:underlay]", func() {
 		if u2oPodNameUnderlay != "" {
 			ginkgo.By("Deleting underlay pod " + u2oPodNameUnderlay)
 			podClient.DeleteSync(u2oPodNameUnderlay)
+		}
 
+		if u2oPodNameOverlay != "" {
 			ginkgo.By("Deleting overlay pod default subnet " + u2oPodNameOverlay)
 			podClient.DeleteSync(u2oPodNameOverlay)
+		}
 
+		if u2oPodOverlayCustomVPC != "" {
 			ginkgo.By("Deleting overlay pod custom vpc " + u2oPodOverlayCustomVPC)
 			podClient.DeleteSync(u2oPodOverlayCustomVPC)
+		}
 
+		if u2oOverlaySubnetNameCustomVPC != "" {
 			ginkgo.By("Deleting subnet in custom vpc" + u2oOverlaySubnetNameCustomVPC)
 			subnetClient.DeleteSync(u2oOverlaySubnetNameCustomVPC)
+		}
 
+		if u2oOverlaySubnetName != "" {
 			ginkgo.By("Deleting subnet " + u2oOverlaySubnetName)
 			subnetClient.DeleteSync(u2oOverlaySubnetName)
+		}
 
+		if vpcName != "" {
 			ginkgo.By("Deleting custom vpc " + vpcName)
 			vpcClient.DeleteSync(vpcName)
 		}
@@ -637,7 +647,39 @@ var _ = framework.SerialDescribe("[group:underlay]", func() {
 		subnet = subnetClient.Get(subnetName)
 		checkU2OItems(f, true, subnet, underlayPod, overlayPod, false)
 
-		ginkgo.By("step9: Disable u2o")
+		f.SkipVersionPriorTo(1, 12, "This feature was introduce in v1.12 ")
+		ginkgo.By("step9: Specify u2oInterconnectionIP ")
+
+		// change u2o interconnection ip twice
+		for i := 0; i < 2; i++ {
+			podClient.DeleteSync(u2oPodNameUnderlay)
+			getAvailableIPs := func(subnet *apiv1.Subnet) string {
+				var availIPs []string
+				if subnet.Status.V4AvailableIPRange != "" {
+					v4AvailIP := strings.Split(strings.Split(subnet.Status.V4AvailableIPRange, ",")[0], "-")[0]
+					availIPs = append(availIPs, v4AvailIP)
+				}
+
+				if subnet.Status.V6AvailableIPRange != "" {
+					v6AvailIP := strings.Split(strings.Split(subnet.Status.V6AvailableIPRange, ",")[0], "-")[0]
+					availIPs = append(availIPs, v6AvailIP)
+				}
+
+				return strings.Join(availIPs, ",")
+			}
+
+			subnet = subnetClient.Get(subnetName)
+			modifiedSubnet = subnet.DeepCopy()
+			modifiedSubnet.Spec.U2OInterconnectionIP = getAvailableIPs(subnet)
+			modifiedSubnet.Spec.U2OInterconnection = true
+			subnetClient.PatchSync(subnet, modifiedSubnet)
+			time.Sleep(5 * time.Second)
+			underlayPod = podClient.CreateSync(originUnderlayPod)
+			subnet = subnetClient.Get(subnetName)
+			checkU2OItems(f, true, subnet, underlayPod, overlayPod, false)
+		}
+
+		ginkgo.By("step10: Disable u2o")
 		podClient.DeleteSync(u2oPodNameUnderlay)
 
 		subnet = subnetClient.Get(subnetName)
@@ -662,12 +704,22 @@ func checkU2OItems(f *framework.Framework, isEnableU2O bool, subnet *apiv1.Subne
 			framework.ExpectEqual(subnet.Status.U2OInterconnectionVPC, subnet.Spec.Vpc)
 		}
 
+		if !f.VersionPriorTo(1, 12) {
+			if subnet.Spec.U2OInterconnectionIP != "" {
+				framework.ExpectEqual(subnet.Spec.U2OInterconnectionIP, subnet.Status.U2OInterconnectionIP)
+			}
+		}
+
 	} else {
 		framework.ExpectFalse(subnet.Spec.U2OInterconnection)
 		framework.ExpectEmpty(subnet.Status.U2OInterconnectionIP)
 		if !f.VersionPriorTo(1, 11) {
 			framework.ExpectEmpty(subnet.Status.U2OInterconnectionVPC)
 		}
+
+		if !f.VersionPriorTo(1, 12) {
+			framework.ExpectEmpty(subnet.Spec.U2OInterconnectionIP)
+		}
 	}
 
 	v4gw, v6gw := util.SplitStringIP(subnet.Spec.Gateway)

yamls/crd.yaml

@@ -2060,6 +2060,8 @@ spec:
                             type: string
                 u2oInterconnection:
                   type: boolean
+                u2oInterconnectionIP:
+                  type: string
                 enableLb:
                   type: boolean
                 enableEcmp: