support hybrid mode for geneve and vlan

cmd/daemon/cniserver.go

@@ -47,7 +47,7 @@ func CmdMain() {
 		klog.Fatalf("init node gateway failed %v", err)
 	}
 
-	if config.NetworkType == util.NetworkTypeVlan {
+	if util.IsNetworkVlan(config.NetworkType) {
 		if err = daemon.InitVlan(config); err != nil {
 			klog.Fatalf("init vlan config failed %v", err)
 		}

docs/vlan.md

@@ -8,8 +8,12 @@ To enable Vlan mode, a dedicated network interface is required by container netw
 The related switch port must work in trunk mode to accept 802.1q packets. For underlay network with no vlan tag, you need
 to set the VLAN ID to 0.
 
-By now, Geneve or Vlan network mode is a global install option, all container must work in the same network mode.
-We are working at combine two networks in one cluster.
+~~By now, Geneve or Vlan network mode is a global install option, all container must work in the same network mode.
+We are working at combine two networks in one cluster.~~
+
+We introduce a new hybrid mode that allows Geneve and Vlan to exist at the same time.
+You can have a subnet A using Geneve encapsulation and subnet B using Vlan tag.
+
 
 ![topology](vlan-topolgy.png "vlan network topology")
 
@@ -71,3 +75,21 @@ spec:
 ```bash
 kubectl run samplepod --image=nginx --namespace=product
 ```
+
+
+### Install Hybrid mode
+
+1. Get the installation script
+
+`wget https://raw.githubusercontent.com/alauda/kube-ovn/release-1.6/dist/images/install.sh`
+
+2. Edit the `install.sh`, modify `NETWORK_TYPE` to `hybrid`, `VLAN_INTERFACE_NAME` to related host interface.
+> NOTE: if your nodes have different nic name for vlan device you could use regex for VLAN_INTERFACE_NAME or label those nodes with
+   own `ovn.kubernetes.io/host_interface_name`.
+
+3. Install Kube-OVN
+
+
+### Note
+Vlan mode will auto-assign a VLAN to a subnet if the subnet doesn't specify a VLAN. 
+The hybrid mode will not do the auto-assign, if your subnet doesn't specify a VLAN then the subnet will treat as Geneve mode.

pkg/controller/config.go

@@ -84,7 +84,7 @@ func ParseFlags() (*Configuration, error) {
 		argWorkerNum = pflag.Int("worker-num", 3, "The parallelism of each worker, default: 3")
 		argPprofPort = pflag.Int("pprof-port", 10660, "The port to get profiling data, default 10660")
 
-		argsNetworkType          = pflag.String("network-type", "geneve", "The ovn network type, default: geneve")
+		argsNetworkType          = pflag.String("network-type", util.NetworkTypeGeneve, "The ovn network type, default: geneve")
 		argsDefaultProviderName  = pflag.String("default-provider-name", "provider", "The vlan or xvlan type default provider interface name, default: provider")
 		argsDefaultInterfaceName = pflag.String("default-interface-name", "", "The default host interface name in the vlan/xvlan type")
 		argsDefaultVlanName      = pflag.String("default-vlan-name", "ovn-vlan", "The default vlan name, default: ovn-vlan")
@@ -139,7 +139,7 @@ func ParseFlags() (*Configuration, error) {
 		PodNamespace:                  os.Getenv("KUBE_NAMESPACE"),
 	}
 
-	if config.NetworkType == util.NetworkTypeVlan && config.DefaultHostInterface == "" {
+	if util.IsNetworkVlan(config.NetworkType) && config.DefaultHostInterface == "" {
 		return nil, fmt.Errorf("no host nic for vlan")
 	}
 

pkg/controller/init.go

@@ -303,7 +303,7 @@ func (c *Controller) InitIPAM() error {
 
 //InitDefaultVlan init the default vlan when network type is vlan or vxlan
 func (c *Controller) initDefaultVlan() error {
-	if c.config.NetworkType != util.NetworkTypeVlan {
+	if !util.IsNetworkVlan(c.config.NetworkType) {
 		return nil
 	}
 

pkg/controller/node.go

@@ -226,13 +226,8 @@ func (c *Controller) handleAddNode(key string) error {
 		}
 	}
 
-	tag, err := c.getSubnetVlanTag(subnet)
-	if err != nil {
-		return err
-	}
-
 	ipStr := util.GetStringIP(v4IP, v6IP)
-	if err := c.ovnClient.CreatePort(c.config.NodeSwitch, portName, ipStr, subnet.Spec.CIDRBlock, mac, tag, "", "", false); err != nil {
+	if err := c.ovnClient.CreatePort(c.config.NodeSwitch, portName, ipStr, subnet.Spec.CIDRBlock, mac, "", "", "",false); err != nil {
 		return err
 	}
 

pkg/controller/pod.go

@@ -12,10 +12,6 @@ import (
 
 	"github.com/intel/multus-cni/logging"
 	multustypes "github.com/intel/multus-cni/types"
-	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
-	"github.com/kubeovn/kube-ovn/pkg/ipam"
-	"github.com/kubeovn/kube-ovn/pkg/ovs"
-	"github.com/kubeovn/kube-ovn/pkg/util"
 	v1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -25,6 +21,11 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/klog"
+
+	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
+	"github.com/kubeovn/kube-ovn/pkg/ipam"
+	"github.com/kubeovn/kube-ovn/pkg/ovs"
+	"github.com/kubeovn/kube-ovn/pkg/util"
 )
 
 func isPodAlive(p *v1.Pod) bool {
@@ -395,8 +396,11 @@ func (c *Controller) handleAddPod(key string) error {
 			return err
 		}
 		ipStr := util.GetStringIP(v4IP, v6IP)
-
-		pod.Annotations[util.NetworkType] = c.config.NetworkType
+		if subnet.Spec.Vlan != "" {
+			pod.Annotations[fmt.Sprintf(util.NetworkTypeTemplate, podNet.ProviderName)] = util.NetworkTypeVlan
+		} else {
+			pod.Annotations[fmt.Sprintf(util.NetworkTypeTemplate, podNet.ProviderName)] = util.NetworkTypeGeneve
+		}
 		pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podNet.ProviderName)] = ipStr
 		pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, podNet.ProviderName)] = mac
 		pod.Annotations[fmt.Sprintf(util.CidrAnnotationTemplate, podNet.ProviderName)] = subnet.Spec.CIDRBlock
@@ -420,7 +424,7 @@ func (c *Controller) handleAddPod(key string) error {
 				pod.Annotations[util.HostInterfaceName] = c.config.DefaultHostInterface
 				pod.Annotations[fmt.Sprintf(util.VlanIdAnnotationTemplate, podNet.ProviderName)] = strconv.Itoa(vlan.Spec.VlanId)
 				pod.Annotations[util.ProviderInterfaceName] = c.config.DefaultProviderName
-				pod.Annotations[util.VlanRangeAnnotation] = c.config.DefaultVlanRange
+				pod.Annotations[fmt.Sprintf(util.VlanRangeAnnotationTemplate, podNet.ProviderName)] = c.config.DefaultVlanRange
 			}
 
 			tag, err := c.getSubnetVlanTag(subnet)

pkg/controller/subnet.go

@@ -86,27 +86,12 @@ func (c *Controller) enqueueUpdateSubnet(old, new interface{}) {
 		oldSubnet.Spec.UnderlayGateway != newSubnet.Spec.UnderlayGateway ||
 		oldSubnet.Spec.Gateway != newSubnet.Spec.Gateway ||
 		!reflect.DeepEqual(oldSubnet.Spec.ExcludeIps, newSubnet.Spec.ExcludeIps) ||
-		!reflect.DeepEqual(oldSubnet.Spec.Vlan, newSubnet.Spec.Vlan) {
+		oldSubnet.Spec.Vlan != newSubnet.Spec.Vlan {
 		klog.V(3).Infof("enqueue update subnet %s", key)
 		c.addOrUpdateSubnetQueue.Add(key)
 	}
 }
 
-func (c *Controller) runAddVpcWorker() {
-	for c.processNextAddVpcWorkItem() {
-	}
-}
-
-func (c *Controller) runUpdateVpcStatusWorker() {
-	for c.processNextUpdateStatusVpcWorkItem() {
-	}
-}
-
-func (c *Controller) runDelVpcWorker() {
-	for c.processNextDeleteVpcWorkItem() {
-	}
-}
-
 func (c *Controller) runAddSubnetWorker() {
 	for c.processNextAddSubnetWorkItem() {
 	}
@@ -275,14 +260,15 @@ func formatSubnet(subnet *kubeovnv1.Subnet, c *Controller) error {
 		if subnet.Spec.Default && subnet.Name != c.config.DefaultLogicalSwitch {
 			subnet.Spec.Default = false
 		}
-		if c.config.NetworkType == util.NetworkTypeVlan && subnet.Spec.Vlan == "" {
+		if util.IsNetworkVlan(c.config.NetworkType) && subnet.Spec.Vlan == "" {
 			subnet.Spec.Vlan = c.config.DefaultVlanName
 			if c.config.DefaultVlanID == 0 {
 				subnet.Spec.UnderlayGateway = true
 			}
 		}
 		if subnet.Spec.Vlan != "" {
 			if _, err := c.vlansLister.Get(subnet.Spec.Vlan); err != nil {
+				klog.Warningf("subnet %s reference a none exist vlan %s", subnet.Name, subnet.Spec.Vlan)
 				subnet.Spec.Vlan = ""
 			}
 		}
@@ -690,7 +676,7 @@ func (c *Controller) handleDeleteLogicalSwitch(key string) error {
 	}
 
 	// re-annotate vlan subnet
-	if c.config.NetworkType == util.NetworkTypeVlan {
+	if util.IsNetworkVlan(c.config.NetworkType) {
 		if err = c.delLocalnet(key); err != nil {
 			return err
 		}
@@ -970,7 +956,7 @@ func (c *Controller) deleteStaticRoute(ip, router string, subnet *kubeovnv1.Subn
 }
 
 func (c *Controller) reconcileVlan(subnet *kubeovnv1.Subnet) error {
-	if c.config.NetworkType != util.NetworkTypeVlan {
+	if !util.IsNetworkVlan(c.config.NetworkType) {
 		return nil
 	}
 

pkg/controller/vlan.go

@@ -224,22 +224,27 @@ func (c *Controller) handleAddVlan(key string) error {
 			return err
 		}
 
-		s.Spec.Vlan = vlan.Name
-		if _, err = c.config.KubeOvnClient.KubeovnV1().Subnets().Update(context.Background(), s, metav1.UpdateOptions{}); err != nil {
-			vlan.Status.SetVlanError("UpdateSubnetVlanFailed", err.Error())
-			bytes, err := vlan.Status.Bytes()
-
-			if err != nil {
-				klog.Error(err)
-			} else {
-				if _, err := c.config.KubeOvnClient.KubeovnV1().Subnets().Patch(context.Background(), vlan.Name, types.MergePatchType, bytes, metav1.PatchOptions{}, "status"); err != nil {
-					klog.Errorf("patch vlan status failed, %v", err)
+		// vlan mode we set vlan for all subnets
+		if c.config.NetworkType == util.NetworkTypeVlan && s.Spec.Vlan == "" {
+			s.Spec.Vlan = vlan.Name
+			if _, err = c.config.KubeOvnClient.KubeovnV1().Subnets().Update(context.Background(), s, metav1.UpdateOptions{}); err != nil {
+				vlan.Status.SetVlanError("UpdateSubnetVlanFailed", err.Error())
+				bytes, err := vlan.Status.Bytes()
+
+				if err != nil {
+					klog.Error(err)
+				} else {
+					if _, err := c.config.KubeOvnClient.KubeovnV1().Subnets().Patch(context.Background(), vlan.Name, types.MergePatchType, bytes, metav1.PatchOptions{}, "status"); err != nil {
+						klog.Errorf("patch vlan status failed, %v", err)
+					}
 				}
+				return err
 			}
-			return err
 		}
 
-		subnets = append(subnets, subnet)
+		if s.Spec.Vlan == vlan.Name {
+			subnets = append(subnets, subnet)
+		}
 	}
 
 	vlan.Spec.Subnet = strings.Join(subnets, ",")
@@ -261,7 +266,7 @@ func (c *Controller) handleUpdateVlan(key string) error {
 		return err
 	}
 
-	if err = util.ValidateVlan(vlan.Spec.VlanId, c.config.DefaultVlanRange); err != nil {
+	if err = util.ValidateVlanTag(vlan.Spec.VlanId, c.config.DefaultVlanRange); err != nil {
 		return err
 	}
 

pkg/controller/vpc.go

@@ -77,6 +77,21 @@ func (c *Controller) enqueueDelVpc(obj interface{}) {
 	}
 }
 
+func (c *Controller) runAddVpcWorker() {
+	for c.processNextAddVpcWorkItem() {
+	}
+}
+
+func (c *Controller) runUpdateVpcStatusWorker() {
+	for c.processNextUpdateStatusVpcWorkItem() {
+	}
+}
+
+func (c *Controller) runDelVpcWorker() {
+	for c.processNextDeleteVpcWorkItem() {
+	}
+}
+
 func (c *Controller) handleDelVpc(vpc *kubeovnv1.Vpc) error {
 	err := c.deleteVpcRouter(vpc.Status.Router)
 	if err != nil {

pkg/daemon/init.go

@@ -58,37 +58,39 @@ func InitMirror(config *Configuration) error {
 }
 
 func InitVlan(config *Configuration) error {
+	// TODO: move to flag validation
+	if config.DefaultProviderName == "" {
+		panic("provider should not be empty")
+	}
 
-	if util.IsProviderVlan(config.NetworkType, config.DefaultProviderName) {
-		//create patch port
-		exists, err := providerBridgeExists()
-		if err != nil {
-			errMsg := fmt.Errorf("check provider bridge exists failed, %v", err)
+	//create patch port
+	exists, err := providerBridgeExists()
+	if err != nil {
+		errMsg := fmt.Errorf("check provider bridge exists failed, %v", err)
+		klog.Error(errMsg)
+		return err
+	}
+
+	if !exists {
+		//create br-provider
+		if err = configProviderPort(config.DefaultProviderName); err != nil {
+			errMsg := fmt.Errorf("configure patch port br-provider failed %v", err)
 			klog.Error(errMsg)
-			return err
+			return errMsg
 		}
 
-		if !exists {
-			//create br-provider
-			if err = configProviderPort(config.DefaultProviderName); err != nil {
-				errMsg := fmt.Errorf("configure patch port br-provider failed %v", err)
-				klog.Error(errMsg)
-				return errMsg
-			}
-
-			//add a host nic to br-provider
-			ifName := config.getInterfaceName()
-			if ifName == "" {
-				errMsg := fmt.Errorf("failed get host nic to add ovs br-provider")
-				klog.Error(errMsg)
-				return errMsg
-			}
+		//add a host nic to br-provider
+		ifName := config.getInterfaceName()
+		if ifName == "" {
+			errMsg := fmt.Errorf("failed get host nic to add ovs br-provider")
+			klog.Error(errMsg)
+			return errMsg
+		}
 
-			if err = configProviderNic(ifName); err != nil {
-				errMsg := fmt.Errorf("add nic %s to port br-provider failed %v", ifName, err)
-				klog.Error(errMsg)
-				return errMsg
-			}
+		if err = configProviderNic(ifName); err != nil {
+			errMsg := fmt.Errorf("add nic %s to port br-provider failed %v", ifName, err)
+			klog.Error(errMsg)
+			return errMsg
 		}
 	}
 	return nil

pkg/util/const.go

@@ -40,6 +40,8 @@ const (
 	IpPoolAnnotationTemplate        = "%s.kubernetes.io/ip_pool"
 	LogicalSwitchAnnotationTemplate = "%s.kubernetes.io/logical_switch"
 	VlanIdAnnotationTemplate        = "%s.kubernetes.io/vlan_id"
+	VlanRangeAnnotationTemplate   = "%s.kubernetes.io/vlan_range"
+	NetworkTypeTemplate          = "%s.kubernetes.io/network_types"
 
 	ExcludeIpsAnnotation = "ovn.kubernetes.io/exclude_ips"
 
@@ -57,10 +59,9 @@ const (
 	ProtocolTCP = "tcp"
 	ProtocolUDP = "udp"
 
-	VlanIdAnnotation      = "ovn.kubernetes.io/vlan_id"
-	VlanRangeAnnotation   = "ovn.kubernetes.io/vlan_range"
-	NetworkType           = "ovn.kubernetes.io/network_types"
 	NetworkTypeVlan       = "vlan"
+	NetworkTypeGeneve     = "geneve"
+	NetworkTypeHybrid     = "hybrid"
 	ProviderInterfaceName = "ovn.kubernetes.io/provider_interface_name"
 	HostInterfaceName     = "ovn.kubernetes.io/host_interface_name"
 

pkg/util/validator.go

@@ -167,7 +167,7 @@ func ValidatePodCidr(cidr, ip string) error {
 	return nil
 }
 
-func ValidateVlan(vlan int, vlanRange string) error {
+func ValidateVlanTag(vlan int, vlanRange string) error {
 	vlans := strings.Split(vlanRange, ",")
 	if len(vlans) != 2 {
 		return fmt.Errorf("the vlan range %s is invalid", vlanRange)