underlay/vlan network refactoring

Changes and new features: 1. Introduce new CRD `ProviderNetwork` for underlay/vlan networking management; 2. Specify provider inetrface on nodes in provider network; 4. Exclude nodes in provider network; 5. Link local IPv6 address and route will NOT be transferred to OVS bridge; 6. Builtin hybrid network support - create/update/delete underlay/vlan networks dynamically; 7. Set MTU of Pod interface to provider interface's MTU on each node; 8. Add new fields in CRD `Vlan`.
kubeovn · Jul 13, 2021 · 2ec0aa7 · 2ec0aa7
1 parent 2441e20
commit 2ec0aa7
Show file tree

Hide file tree

Showing 57 changed files with 3,269 additions and 402 deletions.
diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml
@@ -114,9 +114,9 @@ jobs:
           sudo chmod 666 /home/runner/.kube/config
           make e2e
 
-  single-vlan-e2e:
+  single-vlan-e2e-single-nic:
     needs: build
-    name: 1-master-vlan-e2e
+    name: 1-master-vlan-e2e-single-nic
     runs-on: ubuntu-18.04
     timeout-minutes: 30
     steps:
@@ -159,7 +159,7 @@ jobs:
           go get -u github.com/onsi/gomega/...
           sudo kubectl cluster-info
           sudo chmod 666 /home/runner/.kube/config
-          make e2e-vlan
+          make e2e-vlan-single-nic
 
   single-node-e2e:
     needs: build
@@ -278,13 +278,65 @@ jobs:
           docker load --input image.tar
           sudo make kind-install-ipv6
 
+  ipv6-vlan-e2e-single-nic:
+    needs: build
+    name: ipv6-vlan-e2e-single-nic
+    runs-on: ubuntu-18.04
+    timeout-minutes: 30
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Install Kind
+        env:
+          KIND_VERSION: v0.11.1
+        run: |
+          curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
+          chmod +x ./kind
+          sudo mv kind /usr/local/bin
+
+      - name: Init Kind
+        run: |
+          pip install j2cli --user
+          pip install "j2cli[yaml]" --user
+          sudo PATH=~/.local/bin:$PATH make kind-init-ipv6
+
+      - name: Download image
+        uses: actions/download-artifact@v2
+        with:
+          name: image
+
+      - name: Load Image
+        run: |
+          docker load --input image.tar
+
+      - name: Install Kube-OVN
+        run: |
+          docker load --input image.tar
+          sudo make kind-install-ipv6-vlan
+
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v2
+        with:
+          go-version: ^1.16
+        id: go
+
+      - name: Run E2E
+        run: |
+          go get -u github.com/onsi/ginkgo/ginkgo
+          go get -u github.com/onsi/gomega/...
+          sudo kubectl cluster-info
+          sudo chmod 666 /home/runner/.kube/config
+          make e2e-vlan-single-nic
+
   push:
     needs:
       - single-e2e
-      - single-vlan-e2e
+      - single-vlan-e2e-single-nic
       - single-node-e2e
       - ha-e2e
       - ipv6-e2e
+      - ipv6-vlan-e2e-single-nic
     name: push
     runs-on: ubuntu-18.04
     steps:

diff --git a/Makefile b/Makefile
@@ -159,14 +159,9 @@ kind-install-ipv6:
 
 .PHONY: kind-install-ipv6-vlan
 kind-install-ipv6-vlan:
-	docker network inspect bridge
-	docker network inspect kind
 	$(eval SUBNET = $(shell docker network inspect kind -f "{{(index .IPAM.Config 1).Subnet}}"))
-	$(eval GATEWAY = $(shell docker network inspect kind -f "{{(index .IPAM.Config 1).Gateway}}"))
 	$(eval EXCLUDE_IPS = $(shell docker network inspect kind -f '{{range .Containers}},{{index (split .IPv6Address "/") 0}}{{end}}' | sed 's/^,//'))
-ifeq ($(GATEWAY),)
 	$(eval GATEWAY = $(shell docker exec kube-ovn-worker ip -6 route show default | awk '{print $$3}'))
-endif
 	sed -e 's@^[[:space:]]*POD_CIDR=.*@POD_CIDR="$(SUBNET)"@' \
 		-e 's@^[[:space:]]*POD_GATEWAY=.*@POD_GATEWAY="$(GATEWAY)"@' \
 		-e 's@^[[:space:]]*EXCLUDE_IPS=.*@EXCLUDE_IPS="$(EXCLUDE_IPS)"@' \
@@ -216,13 +211,28 @@ ut:
 
 .PHONY: e2e
 e2e:
+	$(eval NETWORK_BRIDGE = $(shell docker inspect -f '{{json .NetworkSettings.Networks.bridge}}' kube-ovn-control-plane))
+	if [ '$(NETWORK_BRIDGE)' = 'null' ]; then \
+		kind get nodes --name kube-ovn | while read node; do \
+		docker network connect bridge $$node; \
+		done; \
+	fi
+
+	printf "package underlay\n\nvar nodeNetworks = map[string]string{\n" > test/e2e/underlay/network.go
+	kind get nodes --name kube-ovn | while read node; do \
+		printf "\`$$node\`: \`" >> test/e2e/underlay/network.go; \
+		docker inspect -f '{{json .NetworkSettings.Networks.bridge}}' $$node >> test/e2e/underlay/network.go; \
+		printf "\`,\n" >> test/e2e/underlay/network.go; \
+	done
+	echo "}" >> test/e2e/underlay/network.go
+
 	docker pull kubeovn/pause:3.2
 	kind load docker-image --name kube-ovn kubeovn/pause:3.2
 	ginkgo -mod=mod -progress -reportPassed --slowSpecThreshold=60 test/e2e
 
-.PHONY: e2e-vlan
-e2e-vlan:
-	printf "package node\n\nvar networkJSON = []byte(\`" > test/e2e-vlan/node/network.go
-	docker inspect -f '{{json .NetworkSettings.Networks.kind}}' kube-ovn-control-plane >> test/e2e-vlan/node/network.go
-	echo "\`)" >> test/e2e-vlan/node/network.go
-	ginkgo -mod=mod -progress -reportPassed --slowSpecThreshold=60 test/e2e-vlan
+.PHONY: e2e-vlan-single-nic
+e2e-vlan-single-nic:
+	printf "package node\n\nvar networkJSON = []byte(\`" > test/e2e-vlan-single-nic/node/network.go
+	docker inspect -f '{{json .NetworkSettings.Networks.kind}}' kube-ovn-control-plane >> test/e2e-vlan-single-nic/node/network.go
+	echo "\`)" >> test/e2e-vlan-single-nic/node/network.go
+	ginkgo -mod=mod -progress -reportPassed --slowSpecThreshold=60 test/e2e-vlan-single-nic
diff --git a/cmd/daemon/cniserver.go b/cmd/daemon/cniserver.go
@@ -5,22 +5,22 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"k8s.io/apimachinery/pkg/types"
 	"net/http"
 	_ "net/http/pprof" // #nosec
 	"strings"
 	"time"
 
-	"github.com/kubeovn/kube-ovn/pkg/util"
-	"github.com/kubeovn/kube-ovn/versions"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	kubeovninformer "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions"
-	"github.com/kubeovn/kube-ovn/pkg/daemon"
+	"k8s.io/apimachinery/pkg/types"
 	kubeinformers "k8s.io/client-go/informers"
 	"k8s.io/klog"
 	"k8s.io/sample-controller/pkg/signals"
+
+	kubeovninformer "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions"
+	"github.com/kubeovn/kube-ovn/pkg/daemon"
+	"github.com/kubeovn/kube-ovn/pkg/util"
+	"github.com/kubeovn/kube-ovn/versions"
 )
 
 func CmdMain() {
@@ -45,12 +45,6 @@ func CmdMain() {
 		klog.Fatalf("init node gateway failed %v", err)
 	}
 
-	if util.IsNetworkVlan(config.NetworkType) {
-		if err = daemon.InitVlan(config); err != nil {
-			klog.Fatalf("init vlan config failed %v", err)
-		}
-	}
-
 	stopCh := signals.SetupSignalHandler()
 	podInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(config.KubeClient, 0,
 		kubeinformers.WithTweakListOptions(func(listOption *v1.ListOptions) {

diff --git a/dist/images/cleanup.sh b/dist/images/cleanup.sh
@@ -5,6 +5,16 @@ for subnet in $(kubectl get subnet -o name); do
   kubectl patch "$subnet" --type='json' -p '[{"op": "replace", "path": "/metadata/finalizers", "value": []}]'
 done
 
+for vlan in $(kubectl get vlan -o name); do
+  kubectl delete $vlan
+done
+
+for pn in $(kubectl get provider-network -o name); do
+  kubectl delete $pn
+done
+
+sleep 3
+
 # Delete Kube-OVN components
 kubectl delete cm ovn-config ovn-ic-config ovn-external-gw-config -n kube-system --ignore-not-found=true
 kubectl delete secret kube-ovn-tls -n kube-system --ignore-not-found=true
@@ -23,7 +33,14 @@ do
   fi
 done
 kubectl delete ds ovs-ovn kube-ovn-pinger -n kube-system --ignore-not-found=true
-kubectl delete crd ips.kubeovn.io subnets.kubeovn.io vlans.kubeovn.io networks.kubeovn.io --ignore-not-found=true
+kubectl delete crd --ignore-not-found=true \
+  ips.kubeovn.io \
+  subnets.kubeovn.io \
+  vpc-nat-gateways.kubeovn.io \
+  vpcs.kubeovn.io \
+  vlans.kubeovn.io \
+  provider-networks.kubeovn.io \
+  networks.kubeovn.io
 
 # Remove annotations/labels in namespaces and nodes
 kubectl annotate no --all ovn.kubernetes.io/cidr-
@@ -58,8 +75,6 @@ for ns in $(kubectl get ns -o name |cut -c 11-); do
   kubectl annotate pod  --all ovn.kubernetes.io/allocated- -n "$ns"
   kubectl annotate pod  --all ovn.kubernetes.io/routed- -n "$ns"
   kubectl annotate pod  --all ovn.kubernetes.io/vlan_id- -n "$ns"
-  kubectl annotate pod  --all ovn.kubernetes.io/vlan_range- -n "$ns"
-  kubectl annotate pod  --all ovn.kubernetes.io/network_types- -n "$ns"
-  kubectl annotate pod  --all ovn.kubernetes.io/provider_interface_name- -n "$ns"
-  kubectl annotate pod  --all ovn.kubernetes.io/host_interface_name- -n "$ns"
+  kubectl annotate pod  --all ovn.kubernetes.io/network_type- -n "$ns"
+  kubectl annotate pod  --all ovn.kubernetes.io/provider_network- -n "$ns"
 done
diff --git a/dist/images/install-pre-1.16.sh b/dist/images/install-pre-1.16.sh
@@ -44,7 +44,6 @@ PROVIDER_NAME="provider"
 VLAN_INTERFACE_NAME=""
 VLAN_NAME="ovn-vlan"
 VLAN_ID="100"
-VLAN_RANGE="1,4095"
 
 if [ "$ENABLE_VLAN" = "true" ]; then
   NETWORK_TYPE="vlan"
@@ -534,29 +533,112 @@ spec:
     shortNames:
       - vlan
   additionalPrinterColumns:
-    - name: VlanID
+    - name: ID
       type: string
-      JSONPath: .spec.vlanId
-    - name: ProviderInterfaceName
-      type: string
-      JSONPath: .spec.providerInterfaceName
-    - name: Subnet
+      JSONPath: .spec.id
+    - name: Provider
       type: string
-      JSONPath: .spec.subnet
+      JSONPath: .spec.provider
   validation:
     openAPIV3Schema:
       properties:
         spec:
           type: object
           properties:
+            id:
+              type: integer
+              minimum: 0
+              maximum: 4095
+            provider:
+              type: string
             vlanId:
               type: integer
+              description: Deprecated in favor of id
             providerInterfaceName:
               type: string
-            logicalInterfaceName:
-              type: string
-            subnet:
+              description: Deprecated in favor of provider
+          required:
+            - provider
+        status:
+          type: object
+          properties:
+            subnets:
+              type: array
+              items:
+                type: string
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: provider-networks.kubeovn.io
+spec:
+  group: kubeovn.io
+  version: v1
+  scope: Cluster
+  names:
+    plural: provider-networks
+    singular: provider-network
+    kind: ProviderNetwork
+    listKind: ProviderNetworkList
+  additionalPrinterColumns:
+    - name: DefaultInterface
+      type: string
+      jsonPath: .spec.defaultInterface
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          type: object
+          properties:
+            defaultInterface:
               type: string
+            customInterfaces:
+              type: array
+              items:
+                type: object
+                properties:
+                  interface:
+                    type: string
+                  nodes:
+                    type: array
+                    items:
+                      type: string
+            excludeNodes:
+              type: array
+              items:
+                type: string
+          required:
+            - defaultInterface
+        status:
+          type: object
+          properties:
+            readyNodes:
+              type: array
+              items:
+                type: string
+            vlans:
+              type: array
+              items:
+                type: string
+            conditions:
+              type: array
+              items:
+                type: object
+                properties:
+                  node:
+                    type: string
+                  type:
+                    type: string
+                  status:
+                    type: string
+                  reason:
+                    type: string
+                  message:
+                    type: string
+                  lastUpdateTime:
+                    type: string
+                  lastTransitionTime:
+                    type: string
 EOF
 
 if $DPDK; then
@@ -624,6 +706,8 @@ rules:
       - subnets/status
       - ips
       - vlans
+      - provider-networks
+      - provider-networks/status
     verbs:
       - "*"
   - apiGroups:
@@ -1099,6 +1183,8 @@ rules:
       - subnets/status
       - ips
       - vlans
+      - provider-networks
+      - provider-networks/status
       - networks
     verbs:
       - "*"
@@ -2251,6 +2337,7 @@ diagnose(){
   kubectl get crd subnets.kubeovn.io
   kubectl get crd ips.kubeovn.io
   kubectl get crd vlans.kubeovn.io
+  kubectl get crd provider-networks.kubeovn.io
   kubectl get svc kube-dns -n kube-system
   kubectl get svc kubernetes -n default
   kubectl get sa -n kube-system ovn