add ovn-ic HA deploy

dist/images/start-ic-db.sh

@@ -4,8 +4,49 @@ set -eo pipefail
 TS_NAME=${TS_NAME:-ts}
 TS_CIDR=${TS_CIDR:-169.254.100.0/24}
 
-/usr/share/ovn/scripts/ovn-ctl --db-ic-nb-create-insecure-remote=yes --db-ic-sb-create-insecure-remote=yes start_ic_ovsdb
-/usr/share/ovn/scripts/ovn-ctl status_ic_ovsdb
-ovn-ic-nbctl --may-exist ts-add "$TS_NAME"
-ovn-ic-nbctl set Transit_Switch ts external_ids:subnet="$TS_CIDR"
-tail -f /var/log/ovn/ovsdb-server-ic-nb.log
+function quit {
+    /usr/share/ovn/scripts/ovn-ctl stop_ic_ovsdb
+    exit 0
+}
+
+function gen_conn_str {
+    t=$(echo -n "${NODE_IPS}" | sed 's/[[:space:]]//g' | sed 's/,/ /g')
+    x=$(for i in ${t}; do echo -n "tcp:[$i]:$1",; done| sed 's/,$//')
+    echo "$x"
+}
+
+trap quit EXIT
+if [[ -z "$NODE_IPS" && -z "$LOCAL_IP" ]]; then
+    /usr/share/ovn/scripts/ovn-ctl --db-ic-nb-create-insecure-remote=yes --db-ic-sb-create-insecure-remote=yes start_ic_ovsdb
+    /usr/share/ovn/scripts/ovn-ctl status_ic_ovsdb
+    ovn-ic-nbctl --may-exist ts-add "$TS_NAME"
+    ovn-ic-nbctl set Transit_Switch ts external_ids:subnet="$TS_CIDR"
+    tail -f /var/log/ovn/ovsdb-server-ic-nb.log
+else
+    if [[ -z "$LEADER_IP" ]]; then
+        echo "leader start with local ${LOCAL_IP} and cluster $(gen_conn_str 6647)"
+        /usr/share/ovn/scripts/ovn-ctl  --db-ic-nb-create-insecure-remote=yes \
+        --db-ic-sb-create-insecure-remote=yes \
+        --db-ic-sb-cluster-local-addr="${LOCAL_IP}" \
+        --db-ic-nb-cluster-local-addr="${LOCAL_IP}" \
+        --ovn-ic-nb-db="$(gen_conn_str 6647)" \
+        --ovn-ic-sb-db="$(gen_conn_str 6648)" \
+        start_ic_ovsdb
+        /usr/share/ovn/scripts/ovn-ctl status_ic_ovsdb
+        ovn-ic-nbctl --may-exist ts-add "$TS_NAME"
+        ovn-ic-nbctl set Transit_Switch ts external_ids:subnet="$TS_CIDR"
+        tail -f /var/log/ovn/ovsdb-server-ic-nb.log
+    else
+        echo "follower start with local ${LOCAL_IP}, leader ${LEADER_IP} and cluster $(gen_conn_str 6647)"
+        /usr/share/ovn/scripts/ovn-ctl  --db-ic-nb-create-insecure-remote=yes \
+        --db-ic-sb-create-insecure-remote=yes \
+        --db-ic-sb-cluster-local-addr="${LOCAL_IP}" \
+        --db-ic-nb-cluster-local-addr="${LOCAL_IP}" \
+        --db-ic-nb-cluster-remote-addr="${LEADER_IP}" \
+        --db-ic-sb-cluster-remote-addr="${LEADER_IP}" \
+        --ovn-ic-nb-db="$(gen_conn_str 6647)" \
+        --ovn-ic-sb-db="$(gen_conn_str 6648)" \
+        start_ic_ovsdb
+        tail -f /var/log/ovn/ovsdb-server-ic-nb.log
+    fi
+fi

docs/cluster-interconnection.md

@@ -18,7 +18,7 @@ docker run --name=ovn-ic-db -d --network=host -v /etc/ovn/:/etc/ovn -v /var/run/
 ​		If `containerd` replaces `docker` then the command is as follows:
 
 ```shell
-ctr run  -d --net-host --mount="type=bind,src=/etc/ovn/,dst=/etc/ovn,options=rbind:rw" --mount="type=bind,src=/var/run/ovn,dst=/var/run/ovn,options=rbind:rw" --mount="type=bind,src=/var/log/ovn,dst=/var/log/ovn,options=rbind:rw"  kubeovn/kube-ovn:v1.8.6 ovn-ic-db bash start-ic-db.sh
+ctr -n k8s.io run -d --net-host --mount="type=bind,src=/etc/ovn/,dst=/etc/ovn,options=rbind:rw" --mount="type=bind,src=/var/run/ovn,dst=/var/run/ovn,options=rbind:rw" --mount="type=bind,src=/var/log/ovn,dst=/var/log/ovn,options=rbind:rw" docker.io/kubeovn/kube-ovn:v1.8.0 ovn-ic-db bash start-ic-db.sh
 ```
 
 2. Create `ovn-ic-config` ConfigMap in each cluster `kube-system` namespace. Edit and apply the yaml below in each cluster.
@@ -147,7 +147,62 @@ In az2
  kubectl ko nbctl lr-route-add ovn-cluster 10.16.0.0/24 169.254.100.79
 ```
 
+
+
+## Interconnection Controller High Available
+
+1. Run the leader Interconnection Controller in a region that can be accessed by other cluster.  
+
+   `LEADERIP` is the IP of node where leader controller is deployed.
+
+   `NODE_IPS` is the IP of all nodes in the interconnection Controller cluster, in the format `IP1,IP2,...,IPn`.
+
+   `LOCALIP` is the IP of the node on which the container will run. For the leader, `LOCALIP` is its own IP.
+
+```bash
+docker run --name=ovn-ic-db -d --network=host -v /etc/ovn/:/etc/ovn -v /var/run/ovn:/var/run/ovn -v /var/log/ovn:/var/log/ovn -e LOCAL_IP="LEADERIP"  -e NODE_IPS="IP1,IP2,IP3"   kubeovn/kube-ovn:v1.9.0 bash start-ic-db.sh
+```
+
+​        If `containerd` replaces `docker` then the command is as follows:
+
+```shell
+ctr -n k8s.io run -d --net-host --mount="type=bind,src=/etc/ovn/,dst=/etc/ovn,options=rbind:rw" --mount="type=bind,src=/var/run/ovn,dst=/var/run/ovn,options=rbind:rw" --mount="type=bind,src=/var/log/ovn,dst=/var/log/ovn,options=rbind:rw"  --env="NODE_IPS="IP1,IP2,IP3"" --env="LOCAL_IP="LEADERIP"" docker.io/kubeovn/kube-ovn:v1.9.0 ovn-ic-db bash start-ic-db.sh
+```
+
+2. Run the follower Interconnection Controller in the same region. 
+
+```bash
+docker run --name=ovn-ic-db -d --network=host -v /etc/ovn/:/etc/ovn -v /var/run/ovn:/var/run/ovn -v /var/log/ovn:/var/log/ovn -e LOCAL_IP="LOCALIP"  -e NODE_IPS="IP1,IP2,IP3" -e LEADER_IP="LEADERIP"  kubeovn/kube-ovn:v1.9.0 bash start-ic-db.sh
+```
+
+​		If `containerd` replaces `docker` then the command is as follows:
+
+```shell
+ctr -n k8s.io run -d --net-host --mount="type=bind,src=/etc/ovn/,dst=/etc/ovn,options=rbind:rw" --mount="type=bind,src=/var/run/ovn,dst=/var/run/ovn,options=rbind:rw" --mount="type=bind,src=/var/log/ovn,dst=/var/log/ovn,options=rbind:rw" --env="NODE_IPS="IP1,IP2,IP3"" --env="LOCAL_IP="LEADERIP"" --env="NODE_IPS="IP1,IP2,IP3"" docker.io/kubeovn/kube-ovn:v1.9.0 ovn-ic-db bash start-ic-db.sh
+```
+
+3.  Create `ic-config` ConfigMap in each cluster as follows.
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ovn-ic-config
+  namespace: kube-system
+data:
+  enable-ic: "true"
+  az-name: "az1"                												# AZ name for cluster, every cluster should be different
+  ic-db-host: "192.168.65.3,192.168.65.2,192.168.65.1"  # The Interconnection Controller host IP addresses
+  ic-nb-port: "6645"           													# The ic-nb port, default 6645
+  ic-sb-port: "6646"            												# The ic-sb port, default 6646
+  gw-nodes: "az1-gw"            												# The node name which acts as the interconnection gateway
+  auto-route: "false"           # Auto announce route to all clusters. If set false, you can select announced routes later manually
+```
+
+
+
 ## Gateway High Available
+
 Kube-OVN now supports Active-Backup mode gateway HA. You can add more nodes name in the configmap separated by commas.
 
 Active-Active mode gateway HA is under development.

pkg/controller/ovn-ic.go

@@ -92,7 +92,7 @@ func (c *Controller) resyncInterConnection() {
 		if icEnabled == "true" && lastICCM != nil && reflect.DeepEqual(cm.Data, lastICCM) {
 			return
 		}
-		c.ovnClient.OVNIcNBAddress = fmt.Sprintf("%s:%s", cm.Data["ic-db-host"], cm.Data["ic-nb-port"])
+		c.ovnClient.OVNIcNBAddress = genHostAddress(cm.Data["ic-db-host"], cm.Data["ic-nb-port"])
 		klog.Info("start to establish ovn-ic")
 		if err := c.establishInterConnection(cm.Data); err != nil {
 			klog.Errorf("failed to establish ovn-ic, %v", err)
@@ -255,15 +255,15 @@ func (c *Controller) acquireLrpAddress(ts string) (string, error) {
 
 func (c *Controller) startOVNIC(icHost, icNbPort, icSbPort string) error {
 	cmd := exec.Command("/usr/share/ovn/scripts/ovn-ctl",
-		fmt.Sprintf("--ovn-ic-nb-db=tcp:%s:%s", icHost, icNbPort),
-		fmt.Sprintf("--ovn-ic-sb-db=tcp:%s:%s", icHost, icSbPort),
+		fmt.Sprintf("--ovn-ic-nb-db=%s", genHostAddress(icHost, icNbPort)),
+		fmt.Sprintf("--ovn-ic-sb-db=%s", genHostAddress(icHost, icSbPort)),
 		fmt.Sprintf("--ovn-northd-nb-db=%s", c.config.OvnNbAddr),
 		fmt.Sprintf("--ovn-northd-sb-db=%s", c.config.OvnSbAddr),
 		"start_ic")
 	if os.Getenv("ENABLE_SSL") == "true" {
 		cmd = exec.Command("/usr/share/ovn/scripts/ovn-ctl",
-			fmt.Sprintf("--ovn-ic-nb-db=tcp:[%s]:%s", icHost, icNbPort),
-			fmt.Sprintf("--ovn-ic-sb-db=tcp:[%s]:%s", icHost, icSbPort),
+			fmt.Sprintf("--ovn-ic-nb-db=%s", genHostAddress(icHost, icNbPort)),
+			fmt.Sprintf("--ovn-ic-sb-db=%s", genHostAddress(icHost, icSbPort)),
 			fmt.Sprintf("--ovn-northd-nb-db=%s", c.config.OvnNbAddr),
 			fmt.Sprintf("--ovn-northd-sb-db=%s", c.config.OvnSbAddr),
 			"--ovn-ic-ssl-key=/var/run/tls/key",
@@ -344,3 +344,20 @@ func (c *Controller) delLearnedRoute() error {
 	}
 	return nil
 }
+
+func genHostAddress(host string, port string) (hostaddress string) {
+	hostList := strings.Split(host, ",")
+	if len(hostList) == 1 {
+		hostaddress = fmt.Sprintf("tcp:[%s]:%s", hostList[0], port)
+	} else {
+		var blder strings.Builder
+		i := 0
+		for i < len(hostList)-1 {
+			blder.WriteString(fmt.Sprintf("tcp:[%s]:%s,", hostList[i], port))
+			i += 1
+		}
+		blder.WriteString(fmt.Sprintf("tcp:[%s]:%s", hostList[i], port))
+		hostaddress = blder.String()
+	}
+	return hostaddress
+}

pkg/ovs/ovn-ic-nbctl.go

@@ -10,7 +10,7 @@ import (
 
 func (c Client) ovnIcNbCommand(cmdArgs ...string) (string, error) {
 	start := time.Now()
-	cmdArgs = append([]string{fmt.Sprintf("--timeout=%d", c.OvnTimeout), fmt.Sprintf("--db=tcp:%s", c.OVNIcNBAddress)}, cmdArgs...)
+	cmdArgs = append([]string{fmt.Sprintf("--timeout=%d", c.OvnTimeout), fmt.Sprintf("--db=%s", c.OVNIcNBAddress)}, cmdArgs...)
 	raw, err := exec.Command(OVNIcNbCtl, cmdArgs...).CombinedOutput()
 	elapsed := float64((time.Since(start)) / time.Millisecond)
 	klog.V(4).Infof("command %s %s in %vms", OVNIcNbCtl, strings.Join(cmdArgs, " "), elapsed)