support using logical gateway in underlay subnet

dist/images/install-pre-1.16.sh

@@ -7,6 +7,7 @@ ENABLE_VLAN=${ENABLE_VLAN:-false}
 ENABLE_MIRROR=${ENABLE_MIRROR:-false}
 VLAN_NIC=${VLAN_NIC:-}
 CHECK_GATEWAY=${CHECK_GATEWAY:-true}
+LOGICAL_GATEWAY=${LOGICAL_GATEWAY:-false}
 HW_OFFLOAD=${HW_OFFLOAD:-false}
 ENABLE_LB=${ENABLE_LB:-true}
 ENABLE_NP=${ENABLE_NP:-true}
@@ -536,6 +537,8 @@ spec:
               type: boolean
             vlan:
               type: string
+            logicalGateway:
+              type: boolean
             disableGatewayCheck:
               type: boolean
             disableInterConnection:
@@ -1819,6 +1822,7 @@ spec:
           - --default-cidr=$POD_CIDR
           - --default-gateway=$POD_GATEWAY
           - --default-gateway-check=$CHECK_GATEWAY
+          - --default-logical-gateway=$LOGICAL_GATEWAY
           - --default-exclude-ips=$EXCLUDE_IPS
           - --node-switch-cidr=$JOIN_CIDR
           - --service-cluster-ip-range=$SVC_CIDR
@@ -2428,7 +2432,9 @@ trace(){
   fi
 
   gwMac=""
-  if [ ! -z "$(kubectl get subnet $ls -o jsonpath={.spec.vlan})" ]; then
+  vlan=$(kubectl get subnet "$ls" -o jsonpath={.spec.vlan})
+  logicalGateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.logicalGateway})
+  if [ ! -z "$vlan" -a "$logicalGateway" != "true" ]; then
     ovnCni=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep -w kube-ovn-cni | grep " $nodeName " | awk '{print $1}')
     if [ -z "$ovnCni" ]; then
       echo "No kube-ovn-cni Pod running on node $nodeName"

dist/images/install.sh

@@ -6,6 +6,7 @@ DUAL_STACK=${DUAL_STACK:-false}
 ENABLE_SSL=${ENABLE_SSL:-false}
 ENABLE_VLAN=${ENABLE_VLAN:-false}
 CHECK_GATEWAY=${CHECK_GATEWAY:-true}
+LOGICAL_GATEWAY=${LOGICAL_GATEWAY:-false}
 ENABLE_MIRROR=${ENABLE_MIRROR:-false}
 VLAN_NIC=${VLAN_NIC:-}
 HW_OFFLOAD=${HW_OFFLOAD:-false}
@@ -565,6 +566,8 @@ spec:
                   type: boolean
                 vlan:
                   type: string
+                logicalGateway:
+                  type: boolean
                 disableGatewayCheck:
                   type: boolean
                 disableInterConnection:
@@ -1871,6 +1874,7 @@ spec:
           - --default-cidr=$POD_CIDR
           - --default-gateway=$POD_GATEWAY
           - --default-gateway-check=$CHECK_GATEWAY
+          - --default-logical-gateway=$LOGICAL_GATEWAY
           - --default-exclude-ips=$EXCLUDE_IPS
           - --node-switch-cidr=$JOIN_CIDR
           - --service-cluster-ip-range=$SVC_CIDR
@@ -2550,7 +2554,9 @@ trace(){
   fi
 
   gwMac=""
-  if [ ! -z "$(kubectl get subnet $ls -o jsonpath={.spec.vlan})" ]; then
+  vlan=$(kubectl get subnet "$ls" -o jsonpath={.spec.vlan})
+  logicalGateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.logicalGateway})
+  if [ ! -z "$vlan" -a "$logicalGateway" != "true" ]; then
     gateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.gateway})
     if [[ "$gateway" =~ .*,.* ]]; then
       if [ "$af" = "4" ]; then

dist/images/kubectl-ko

@@ -121,7 +121,9 @@ trace(){
   fi
 
   gwMac=""
-  if [ ! -z "$(kubectl get subnet $ls -o jsonpath={.spec.vlan})" ]; then
+  vlan=$(kubectl get subnet "$ls" -o jsonpath={.spec.vlan})
+  logicalGateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.logicalGateway})
+  if [ ! -z "$vlan" -a "$logicalGateway" != "true" ]; then
     gateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.gateway})
     if [[ "$gateway" =~ .*,.* ]]; then
       if [ "$af" = "4" ]; then

docs/multi-nic.md

@@ -77,7 +77,7 @@ spec:
   - 172.17.0.1..172.17.0.10
 ```
 
-Other options like gatewayType, gatewayNode, private, allowSubnets, vlan, natOutgoing are not available for attachment network.
+Other options like gatewayType, gatewayNode, private, allowSubnets, vlan, logicalGateway, natOutgoing are not available for attachment network.
 
 ### Create Pod with multi network
 

docs/subnet.md

@@ -66,6 +66,7 @@ Since kube-ovn v1.8.0, kube-ovn support using designative egress ip on node, the
 ## Advance Options
 
 - `vlan`: if enable vlan network, use this field to specific which vlan the subnet should bind to.
+- `logicalGateway`: Create a logical gateway for the subnet instead of using underlay gateway. Take effect only when the subnet is in underlay mode. Default: `false`.
 - `externalEgressGateway`: External egress gateway address. When set, egress traffic is redirected to the external gateway through gateway node(s) by policy-based routing. Conflict with `natOutgoing`.
 - `policyRoutingPriority`/`policyRoutingTableID`: Priority & table ID used in policy-based routing. Required when `externalEgressGateway` is set. NOTICE: `policyRoutingTableID` MUST be unique.
 - `disableGatewayCheck`: By default Kube-OVN checks Pod's network by sending ICMP request to the subnet's gateway. Set it to `true` if the subnet is in underlay mode and the physical gateway does not respond to ICMP requests.

pkg/apis/kubeovn/v1/types.go

@@ -121,6 +121,7 @@ type SubnetSpec struct {
 	Vlan   string `json:"vlan,omitempty"`
 	HtbQos string `json:"htbqos,omitempty"`
 
+	LogicalGateway         bool `json:"logicalGateway"`
 	DisableGatewayCheck    bool `json:"disableGatewayCheck"`
 	DisableInterConnection bool `json:"disableInterConnection"`
 }

pkg/controller/config.go

@@ -34,11 +34,12 @@ type Configuration struct {
 	KubeFactoryClient    kubernetes.Interface
 	KubeOvnFactoryClient clientset.Interface
 
-	DefaultLogicalSwitch string
-	DefaultCIDR          string
-	DefaultGateway       string
-	DefaultExcludeIps    string
-	DefaultGatewayCheck  bool
+	DefaultLogicalSwitch  string
+	DefaultCIDR           string
+	DefaultGateway        string
+	DefaultExcludeIps     string
+	DefaultGatewayCheck   bool
+	DefaultLogicalGateway bool
 
 	ClusterRouter     string
 	NodeSwitch        string
@@ -79,11 +80,12 @@ func ParseFlags() (*Configuration, error) {
 		argOvnTimeout     = pflag.Int("ovn-timeout", 60, "")
 		argKubeConfigFile = pflag.String("kubeconfig", "", "Path to kubeconfig file with authorization and master location information. If not set use the inCluster token.")
 
-		argDefaultLogicalSwitch = pflag.String("default-ls", "ovn-default", "The default logical switch name, default: ovn-default")
-		argDefaultCIDR          = pflag.String("default-cidr", "10.16.0.0/16", "Default CIDR for namespace with no logical switch annotation, default: 10.16.0.0/16")
-		argDefaultGateway       = pflag.String("default-gateway", "", "Default gateway for default-cidr, default the first ip in default-cidr")
-		argDefaultGatewayCheck  = pflag.Bool("default-gateway-check", true, "Check switch for the default subnet's gateway, default: true")
-		argDefaultExcludeIps    = pflag.String("default-exclude-ips", "", "Exclude ips in default switch, default equals to gateway address")
+		argDefaultLogicalSwitch  = pflag.String("default-ls", "ovn-default", "The default logical switch name, default: ovn-default")
+		argDefaultCIDR           = pflag.String("default-cidr", "10.16.0.0/16", "Default CIDR for namespace with no logical switch annotation, default: 10.16.0.0/16")
+		argDefaultGateway        = pflag.String("default-gateway", "", "Default gateway for default-cidr, default the first ip in default-cidr")
+		argDefaultGatewayCheck   = pflag.Bool("default-gateway-check", true, "Check switch for the default subnet's gateway, default: true")
+		argDefaultLogicalGateway = pflag.Bool("default-logical-gateway", false, "Create a logical gateway for the default subnet instead of using underlay gateway. Take effect only when the default subnet is in underlay mode.")
+		argDefaultExcludeIps     = pflag.String("default-exclude-ips", "", "Exclude ips in default switch, default equals to gateway address")
 
 		argClusterRouter     = pflag.String("cluster-router", "ovn-cluster", "The router name for cluster router, default: ovn-cluster")
 		argNodeSwitch        = pflag.String("node-switch", "join", "The name of node gateway switch which help node to access pod network, default: join")
@@ -138,6 +140,7 @@ func ParseFlags() (*Configuration, error) {
 		DefaultCIDR:                   *argDefaultCIDR,
 		DefaultGateway:                *argDefaultGateway,
 		DefaultGatewayCheck:           *argDefaultGatewayCheck,
+		DefaultLogicalGateway:         *argDefaultLogicalGateway,
 		DefaultExcludeIps:             *argDefaultExcludeIps,
 		ClusterRouter:                 *argClusterRouter,
 		NodeSwitch:                    *argNodeSwitch,

pkg/controller/init.go

@@ -127,6 +127,7 @@ func (c *Controller) initDefaultLogicalSwitch() error {
 	}
 	if c.config.NetworkType == util.NetworkTypeVlan {
 		defaultSubnet.Spec.Vlan = c.config.DefaultVlanName
+		defaultSubnet.Spec.LogicalGateway = c.config.DefaultLogicalGateway
 	}
 
 	_, err = c.config.KubeOvnClient.KubeovnV1().Subnets().Create(context.Background(), &defaultSubnet, metav1.CreateOptions{})

pkg/controller/node.go

@@ -536,7 +536,9 @@ func (c *Controller) checkGatewayReady() error {
 	}
 
 	for _, subnet := range subnetList {
-		if subnet.Spec.Vlan != "" || subnet.Spec.GatewayType != kubeovnv1.GWCentralizedType || subnet.Spec.GatewayNode == "" {
+		if (subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway) ||
+			subnet.Spec.GatewayNode == "" ||
+			subnet.Spec.GatewayType != kubeovnv1.GWCentralizedType {
 			continue
 		}
 

pkg/controller/subnet.go

@@ -87,6 +87,7 @@ func (c *Controller) enqueueUpdateSubnet(old, new interface{}) {
 		!reflect.DeepEqual(oldSubnet.Spec.Namespaces, newSubnet.Spec.Namespaces) ||
 		oldSubnet.Spec.GatewayType != newSubnet.Spec.GatewayType ||
 		oldSubnet.Spec.GatewayNode != newSubnet.Spec.GatewayNode ||
+		oldSubnet.Spec.LogicalGateway != newSubnet.Spec.LogicalGateway ||
 		oldSubnet.Spec.Gateway != newSubnet.Spec.Gateway ||
 		!reflect.DeepEqual(oldSubnet.Spec.ExcludeIps, newSubnet.Spec.ExcludeIps) ||
 		oldSubnet.Spec.Vlan != newSubnet.Spec.Vlan {
@@ -569,7 +570,7 @@ func (c *Controller) handleAddOrUpdateSubnet(key string) error {
 		return err
 	}
 
-	needRouter := subnet.Spec.Vlan == "" || subnet.Spec.Vpc != util.DefaultVpc
+	needRouter := (subnet.Spec.Vlan == "" || subnet.Spec.LogicalGateway) || subnet.Spec.Vpc != util.DefaultVpc
 	if !exist {
 		subnet.Status.EnsureStandardConditions()
 		// If multiple namespace use same ls name, only first one will success
@@ -857,13 +858,15 @@ func (c *Controller) reconcileGateway(subnet *kubeovnv1.Subnet) error {
 			return err
 		}
 
-		if err := c.ovnClient.DeleteLogicalSwitchPort(fmt.Sprintf("%s-%s", subnet.Name, c.config.ClusterRouter)); err != nil {
-			klog.Errorf("failed to delete lsp %s-%s, %v", subnet.Name, c.config.ClusterRouter, err)
-			return err
-		}
-		if err := c.ovnClient.DeleteLogicalRouterPort(fmt.Sprintf("%s-%s", c.config.ClusterRouter, subnet.Name)); err != nil {
-			klog.Errorf("failed to delete lrp %s-%s, %v", c.config.ClusterRouter, subnet.Name, err)
-			return err
+		if !subnet.Spec.LogicalGateway {
+			if err := c.ovnClient.DeleteLogicalSwitchPort(fmt.Sprintf("%s-%s", subnet.Name, c.config.ClusterRouter)); err != nil {
+				klog.Errorf("failed to delete lsp %s-%s, %v", subnet.Name, c.config.ClusterRouter, err)
+				return err
+			}
+			if err := c.ovnClient.DeleteLogicalRouterPort(fmt.Sprintf("%s-%s", c.config.ClusterRouter, subnet.Name)); err != nil {
+				klog.Errorf("failed to delete lrp %s-%s, %v", c.config.ClusterRouter, subnet.Name, err)
+				return err
+			}
 		}
 	} else {
 		// if gw is distributed remove activateGateway field

test/e2e/subnet/normal.go

@@ -604,73 +604,4 @@ var _ = Describe("[Subnet]", func() {
 			Expect(stdout).NotTo(HavePrefix(routePrefix))
 		})
 	})
-
-	Describe("Disable Gateway Check", func() {
-		It("disable gateway check", func() {
-			name := f.GetName()
-
-			By("create namespace")
-			namespace := &corev1.Namespace{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:   name,
-					Labels: map[string]string{"e2e": "true"},
-				},
-			}
-			_, err := f.KubeClientSet.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{})
-			Expect(err).NotTo(HaveOccurred())
-
-			By("create subnet")
-			subnet := &kubeovn.Subnet{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:   name,
-					Labels: map[string]string{"e2e": "true"},
-				},
-				Spec: kubeovn.SubnetSpec{
-					CIDRBlock:           "11.17.0.0/16",
-					Namespaces:          []string{namespace.Name},
-					DisableGatewayCheck: true,
-				},
-			}
-			_, err = f.OvnClientSet.KubeovnV1().Subnets().Create(context.Background(), subnet, metav1.CreateOptions{})
-			Expect(err).NotTo(HaveOccurred())
-
-			By("validate subnet")
-			err = f.WaitSubnetReady(subnet.Name)
-			Expect(err).NotTo(HaveOccurred())
-
-			By("create pod")
-			pod := &corev1.Pod{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:      name,
-					Namespace: namespace.Name,
-				},
-				Spec: corev1.PodSpec{
-					Containers: []corev1.Container{
-						{
-							Name:            name,
-							Image:           "kubeovn/pause:3.2",
-							ImagePullPolicy: corev1.PullIfNotPresent,
-						},
-					},
-				},
-			}
-			_, err = f.KubeClientSet.CoreV1().Pods(pod.Namespace).Create(context.Background(), pod, metav1.CreateOptions{})
-			Expect(err).NotTo(HaveOccurred())
-
-			_, err = f.WaitPodReady(pod.Name, pod.Namespace)
-			Expect(err).NotTo(HaveOccurred())
-
-			By("delete pod")
-			err = f.KubeClientSet.CoreV1().Pods(pod.Namespace).Delete(context.Background(), pod.Name, metav1.DeleteOptions{})
-			Expect(err).NotTo(HaveOccurred())
-
-			By("delete subnet")
-			err = f.OvnClientSet.KubeovnV1().Subnets().Delete(context.Background(), subnet.Name, metav1.DeleteOptions{})
-			Expect(err).NotTo(HaveOccurred())
-
-			By("delete namespace")
-			err = f.KubeClientSet.CoreV1().Namespaces().Delete(context.Background(), namespace.Name, metav1.DeleteOptions{})
-			Expect(err).NotTo(HaveOccurred())
-		})
-	})
 })

test/e2e/underlay/underlay.go

@@ -173,6 +173,112 @@ var _ = Describe("[Underlay]", func() {
 		})
 	})
 
+	Context("[Subnet]", func() {
+		BeforeEach(func() {
+			err := f.OvnClientSet.KubeovnV1().Subnets().Delete(context.Background(), f.GetName(), metav1.DeleteOptions{})
+			if err != nil && !k8serrors.IsNotFound(err) {
+				klog.Fatalf("failed to delete subnet %s: %v", f.GetName(), err)
+			}
+		})
+		AfterEach(func() {
+			err := f.OvnClientSet.KubeovnV1().Subnets().Delete(context.Background(), f.GetName(), metav1.DeleteOptions{})
+			if err != nil && !k8serrors.IsNotFound(err) {
+				klog.Fatalf("failed to delete subnet %s: %v", f.GetName(), err)
+			}
+		})
+
+		It("logical gateway", func() {
+			name := f.GetName()
+
+			By("create subnet")
+			subnet := &kubeovn.Subnet{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:   name,
+					Labels: map[string]string{"e2e": "true"},
+				},
+				Spec: kubeovn.SubnetSpec{
+					CIDRBlock:      "99.11.0.0/16",
+					Vlan:           Vlan,
+					LogicalGateway: true,
+				},
+			}
+			_, err := f.OvnClientSet.KubeovnV1().Subnets().Create(context.Background(), subnet, metav1.CreateOptions{})
+			Expect(err).NotTo(HaveOccurred())
+
+			By("validate subnet")
+			err = f.WaitSubnetReady(subnet.Name)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("validate OVN logical router port")
+			ovnPods, err := f.KubeClientSet.CoreV1().Pods("kube-system").List(context.Background(), metav1.ListOptions{LabelSelector: "app=ovn-central"})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(ovnPods).NotTo(BeNil())
+
+			ovnPod := ovnPods.Items[0]
+			lsp := fmt.Sprintf("%s-%s", name, util.DefaultVpc)
+			cmd := fmt.Sprintf("ovn-nbctl --no-heading --columns=_uuid find logical_switch_port name=%s", lsp)
+			uuid, _, err := f.ExecToPodThroughAPI(cmd, "ovn-central", ovnPod.Name, ovnPod.Namespace, nil)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(uuid).NotTo(BeEmpty())
+
+			lrp := fmt.Sprintf("%s-%s", util.DefaultVpc, name)
+			cmd = fmt.Sprintf("ovn-nbctl --no-heading --columns=_uuid find logical_router_port name=%s", lrp)
+			uuid, _, err = f.ExecToPodThroughAPI(cmd, "ovn-central", ovnPod.Name, ovnPod.Namespace, nil)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(uuid).NotTo(BeEmpty())
+		})
+
+		It("disable gateway check", func() {
+			name := f.GetName()
+
+			By("create subnet")
+			subnet := &kubeovn.Subnet{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:   name,
+					Labels: map[string]string{"e2e": "true"},
+				},
+				Spec: kubeovn.SubnetSpec{
+					CIDRBlock:           "99.12.0.0/16",
+					Vlan:                Vlan,
+					DisableGatewayCheck: true,
+				},
+			}
+			_, err := f.OvnClientSet.KubeovnV1().Subnets().Create(context.Background(), subnet, metav1.CreateOptions{})
+			Expect(err).NotTo(HaveOccurred())
+
+			By("validate subnet")
+			err = f.WaitSubnetReady(subnet.Name)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("create pod")
+			pod := &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:        name,
+					Namespace:   Namespace,
+					Annotations: map[string]string{util.LogicalSwitchAnnotation: name},
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name:            name,
+							Image:           "kubeovn/pause:3.2",
+							ImagePullPolicy: corev1.PullIfNotPresent,
+						},
+					},
+				},
+			}
+			_, err = f.KubeClientSet.CoreV1().Pods(pod.Namespace).Create(context.Background(), pod, metav1.CreateOptions{})
+			Expect(err).NotTo(HaveOccurred())
+
+			_, err = f.WaitPodReady(pod.Name, pod.Namespace)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("delete pod")
+			err = f.KubeClientSet.CoreV1().Pods(pod.Namespace).Delete(context.Background(), pod.Name, metav1.DeleteOptions{})
+			Expect(err).NotTo(HaveOccurred())
+		})
+	})
+
 	Context("[Pod]", func() {
 		var cniPods map[string]corev1.Pod
 		BeforeEach(func() {