netmaster cli refactor (#1088)

* netmaster cli refactor Refactor netmaster cli to use "github.com/urfave/cli", it brings benefit like setting forarding mode, infra type and network mode, allows setting multi etcd/consul endpoints and using SSL, enabling syslog,json log and multi log levels. Signed-off-by: Wei Tie <nuaafe@gmail.com>
contiv · Dec 5, 2017 · 43d7b3f · 43d7b3f
1 parent 0b11750
commit 43d7b3f
Show file tree

Hide file tree

Showing 38 changed files with 848 additions and 689 deletions.
diff --git a/Vagrantfile b/Vagrantfile
@@ -24,7 +24,8 @@ https_proxy = ENV['HTTPS_PROXY'] || ENV['https_proxy'] || ''
 build_version = ENV['BUILD_VERSION'] || ''
 cluster_ip_nodes = ''
 v2plugin_name = ENV['CONTIV_V2PLUGIN_NAME'] || 'contiv/v2netplugin:0.1'
-cluster_store = ENV['CONTIV_CLUSTER_STORE'] || 'etcd://localhost:2379'
+cluster_store_driver = ENV['CONTIV_CLUSTER_STORE_DRIVER'] || 'etcd'
+cluster_store_url = ENV['CONTIV_CLUSTER_STORE_URL'] || 'http://localhost:2379'
 nightly_release = ENV['NIGHTLY_RELEASE'] || ''
 node_os = ENV['CONTIV_NODE_OS'] || 'centos'
 base_ip = ENV['CONTIV_IP_PREFIX'] || '192.168.2.'
@@ -46,7 +47,8 @@ export https_proxy='#{https_proxy}'
 export NIGHTLY_RELEASE=#{nightly_release}
 export no_proxy=%{cluster_ip_nodes},127.0.0.1,localhost,netmaster
 export CLUSTER_NODE_IPS=%{cluster_ip_nodes}
-export CONTIV_CLUSTER_STORE=#{cluster_store}
+export CONTIV_CLUSTER_STORE_DRIVER=#{cluster_store_driver}
+export CONTIV_CLUSTER_STORE_URL=#{cluster_store_url}
 export CONTIV_V2PLUGIN_NAME=#{v2plugin_name}
 export CONTIV_DOCKER_SWARM=#{docker_swarm}
 export BUILD_VERSION=#{build_version}
@@ -119,7 +121,7 @@ fi
 if [[ #{docker_swarm} == "swarm_mode" ]]; then
     perl -i -lpe 's!^ExecStart(.+)$!ExecStart$1 !' /lib/systemd/system/docker.service
 else
-    if [[ "$CONTIV_CLUSTER_STORE" == *"consul:"* ]]
+    if [[ "$CONTIV_CLUSTER_STORE_DRIVER" == "consul" ]]
     then
         perl -i -lpe 's!^ExecStart(.+)$!ExecStart$1 -H tcp://0.0.0.0:2375 -H unix:///var/run/docker.sock --cluster-store=consul://localhost:8500!' /lib/systemd/system/docker.service
     else
@@ -225,7 +227,7 @@ module VagrantPlugins
 
       def self.up_hook(arg)
         unless File.exist?(STATEFILE) # prevent it from writing more than once.
-          f = File.open(STATEFILE, "w") 
+          f = File.open(STATEFILE, "w")
           ENV.each do |x,y|
             f.puts "%s=%s" % [x,y]
           end
@@ -256,7 +258,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     config.vm.box_check_update = false
     if Vagrant.has_plugin?("vagrant-vbguest")
         config.vbguest.auto_update = false
-    end 
+    end
     if node_os == "ubuntu" then
         config.vm.box = "contiv/ubuntu1604-netplugin"
         config.vm.box_version = "0.7.0"

diff --git a/core/common.go b/core/common.go
@@ -0,0 +1,9 @@
+package core
+
+// Plugin modes
+const (
+	Docker     = "docker"
+	Kubernetes = "kubernetes"
+	SwarmMode  = "swarm-mode"
+	Test       = "test"
+)
diff --git a/core/error_test.go b/core/error_test.go
@@ -34,7 +34,7 @@ func TestErrorStackTrace(t *testing.T) {
 	}
 
 	if e.Error() == "an error\n" {
-		t.Fatal("Error message did not yield stack trace with CONTIV_TRACE set")
+		t.Fatal("Error message did not yield stack trace")
 	}
 
 	lines := strings.Split(e.Error(), "\n")

diff --git a/install/HowtoSetupContiv.md b/install/HowtoSetupContiv.md
@@ -14,33 +14,52 @@ Download tarball for the version you want to try.
 We recommend starting netmaster on master node in your master slave architecture. You can start netmaster in HA mode and appropriate netmaster node will handle all the queries from end user.
 
 ```
-netmaster --help
-Usage: netmaster [OPTION]...
-  -cluster-mode string
-        {docker, kubernetes} (default "docker")
-  -cluster-store string
-        Etcd or Consul cluster store url. (default "etcd://127.0.0.1:2379") << we support consul as well, you need to change it here.
-  -debug
-        Turn on debugging information
-  -help
-        prints this message
-  -listen-url string
-        Url to listen http requests on (default ":9999")
-  -version
-        prints current version
+netplugin --help
+NAME:
+   netplugin - Contiv netplugin service
+
+USAGE:
+   netplugin [global options] command [command options] [arguments...]
+
+VERSION:
+
+Version: <netplugin-version>
+GitCommit: <netplugin-commit-sha>
+BuildTime: <netplugin-build-time>
+
+
+COMMANDS:
+     help, h  Shows a list of commands or help for one command
+
+GLOBAL OPTIONS:
+   --consul-endpoints value, --consul value                 a comma-delimited list of netplugin consul endpoints [$CONTIV_NETPLUGIN_CONSUL_ENDPOINTS]
+   --ctrl-ip value                                          set netplugin control ip for control plane communication (default: <host-ip-from-local-resolver>) [$CONTIV_NETPLUGIN_CONTROL_IP]
+   --etcd-endpoints value, --etcd value                     a comma-delimited list of netplugin etcd endpoints [$CONTIV_NETPLUGIN_ETCD_ENDPOINTS]
+   --fwdmode value, --forward-mode value                    set netplugin forwarding network mode, options: [bridge, routing] [$CONTIV_NETPLUGIN_FORWARD_MODE]
+   --host value, --host-label value                         set netplugin host to identify itself (default: <host-name-reported-by-the-kernel>) [$CONTIV_NETPLUGIN_HOST]
+   --log-level value                                        set netplugin log level, options: [DEBUG, INFO, WARN, ERROR] (default: "INFO") [$CONTIV_NETPLUGIN_LOG_LEVEL]
+   --mode value, --plugin-mode value, --cluster-mode value  set netplugin mode, options: [docker, kubernetes, swarm-mode] [$CONTIV_NETPLUGIN_MODE]
+   --netmode value, --network-mode value                    set netplugin network mode, options: [vlan, vxlan] [$CONTIV_NETPLUGIN_NET_MODE]
+   --syslog-url value                                       set netplugin syslog url in format protocol://ip:port (default: "udp://127.0.0.1:514") [$CONTIV_NETPLUGIN_SYSLOG_URL]
+   --use-json-log, --json-log                               set netplugin log format to json if this flag is provided [$CONTIV_NETPLUGIN_USE_JSON_LOG]
+   --use-syslog, --syslog                                   set netplugin send log to syslog if this flag is provided [$CONTIV_NETPLUGIN_USE_SYSLOG]
+   --vlan-uplinks value, --vlan-if value                    a comma-delimited list of netplugin uplink interfaces [$CONTIV_NETPLUGIN_VLAN_UPLINKS]
+   --vtep-ip value                                          set netplugin vtep ip for vxlan communication (default: <host-ip-from-local-resolver>) [$CONTIV_NETPLUGIN_VTEP_IP]
+   --vxlan-port value                                       set netplugin VXLAN port (default: 4789) [$CONTIV_NETPLUGIN_VXLAN_PORT]
+   --help, -h                                               show help
+   --version, -v                                            print the version
 ```
 
 ## How to start netplugin ?
 
 You start netplugin on each node in your cluster.
 
 ```
-netplugin --help
 NAME:
-   netplugin - Contiv netplugin service
+   netmaster - Contiv netmaster service
 
 USAGE:
-   netplugin [global options] command [command options] [arguments...]
+   netmaster [global options] command [command options] [arguments...]
 
 VERSION:
 
@@ -53,22 +72,21 @@ COMMANDS:
      help, h  Shows a list of commands or help for one command
 
 GLOBAL OPTIONS:
-   --consul-endpoints value, --consul value  set netplugin consul endpoints [$CONTIV_NETPLUGIN_CONSUL_ENDPOINTS]
-   --ctrl-ip value                           set netplugin control ip for control plane communication (default: <current-ip-address>) [$CONTIV_NETPLUGIN_CONTROL_IP]
-   --etcd-endpoints value, --etcd value      set netplugin etcd endpoints [$CONTIV_NETPLUGIN_ETCD_ENDPOINTS]
-   --fwdmode value, --forward-mode value     set netplugin forwarding network mode, options: [bridge, routing] [$CONTIV_NETPLUGIN_NET_MODE]
-   --host value, --host-label value          set netplugin host to identify itself (default: "8994bb27e5fb") [$CONTIV_NETPLUGIN_HOST]
-   --log-level value                         set netplugin log level, options: [DEBUG, INFO, WARN, ERROR] (default: "INFO") [$CONTIV_NETPLUGIN_LOG_LEVEL]
-   --mode value, --plugin-mode value         set netplugin mode, options: [docker, kubernetes, swarm-mode] [$CONTIV_NETPLUGIN_MODE]
-   --netmode value, --network-mode value     set netplugin network mode, options: [vlan, vxlan] [$CONTIV_NETPLUGIN_NET_MODE]
-   --syslog-url value                        set netplugin syslog url in format protocol://ip:port (default: "udp://127.0.0.1:514") [$CONTIV_NETPLUGIN_SYSLOG_URL]
-   --use-json-log, --json-log                set netplugin log format to json [$CONTIV_NETPLUGIN_USE_JSON_LOG]
-   --use-syslog, --syslog                    set netplugin send log to syslog or not [$CONTIV_NETPLUGIN_USE_SYSLOG]
-   --vlan-uplinks value, --vlan-if value     set netplugin uplink interfaces [$CONTIV_NETPLUGIN_VLAN_UPLINKS]
-   --vtep-ip value                           set netplugin vtep ip for vxlan communication (default: <current-ip-address>) [$CONTIV_NETPLUGIN_VTEP_IP]
-   --vxlan-port value                        set netplugin VXLAN port (default: 4789) [$CONTIV_NETPLUGIN_VXLAN_PORT]
-   --help, -h                                show help
-   --version, -v                             print the version
+   --consul-endpoints value, --consul value                 a comma-delimited list of netmaster consul endpoints [$CONTIV_NETMASTER_CONSUL_ENDPOINTS]
+   --etcd-endpoints value, --etcd value                     a comma-delimited list of netmaster etcd endpoints [$CONTIV_NETMASTER_ETCD_ENDPOINTS]
+   --external-address value, --listen-url value             set netmaster external address to listen on, used for general API service (default: "0.0.0.0:9999") [$CONTIV_NETMASTER_EXTERNAL_ADDRESS]
+   --fwdmode value, --forward-mode value                    set netmaster forwarding network mode, options: [bridge, routing] [$CONTIV_NETMASTER_FORWARD_MODE]
+   --infra value, --infra-type value                        set netmaster infra type, options [aci, default] (default: "default") [$CONTIV_NETMASTER_INFRA]
+   --internal-address value, --control-url value            set netmaster internal address to listen on, used for RPC and leader election (default: <host-ip-from-local-resolver>:<port-of-external-address>) [$CONTIV_NETMASTER_INTERNAL_ADDRESS]
+   --log-level value                                        set netmaster log level, options: [DEBUG, INFO, WARN, ERROR] (default: "INFO") [$CONTIV_NETMASTER_LOG_LEVEL]
+   --mode value, --plugin-mode value, --cluster-mode value  set netmaster mode, options: [docker, kubernetes, swarm-mode] [$CONTIV_NETMASTER_MODE]
+   --name value, --plugin-name value                        set netmaster plugin name for docker v2 plugin (default: "netplugin") [$CONTIV_NETMASTER_PLUGIN_NAME]
+   --netmode value, --network-mode value                    set netmaster network mode, options: [vlan, vxlan] [$CONTIV_NETMASTER_NET_MODE]
+   --syslog-url value                                       set netmaster syslog url in format protocol://ip:port (default: "udp://127.0.0.1:514") [$CONTIV_NETMASTER_SYSLOG_URL]
+   --use-json-log, --json-log                               set netmaster log format to json if this flag is provided [$CONTIV_NETMASTER_USE_JSON_LOG]
+   --use-syslog, --syslog                                   set netmaster send log to syslog if this flag is provided [$CONTIV_NETMASTER_USE_SYSLOG]
+   --help, -h                                               show help
+   --version, -v                                            print the version
 ```
 
 for example we can start netplugin in following manner :

diff --git a/mgmtfn/dockplugin/netDriver.go b/mgmtfn/dockplugin/netDriver.go
@@ -25,6 +25,7 @@ import (
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/contiv/netplugin/contivmodel/client"
+	"github.com/contiv/netplugin/core"
 	"github.com/contiv/netplugin/netmaster/docknet"
 	"github.com/contiv/netplugin/netmaster/intent"
 	"github.com/contiv/netplugin/netmaster/master"
@@ -399,7 +400,7 @@ func allocateNetwork(w http.ResponseWriter, r *http.Request) {
 	// node where a container in the network is instantiated.
 	// We process only allocateNetwork in this case.
 	//
-	if pluginMode == master.SwarmMode {
+	if pluginMode == core.SwarmMode {
 		tag := ""
 		log.Infof("Options: %+v", anreq.Options)
 		if _, tagOk := anreq.Options["contiv-tag"]; tagOk {
@@ -441,7 +442,7 @@ func freeNetwork(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if pluginMode == master.SwarmMode {
+	if pluginMode == core.SwarmMode {
 		err = deleteNetworkHelper(req.NetworkID)
 		if err != nil {
 			httpError(w, "Could not delete network", err)
@@ -536,7 +537,7 @@ func GetDockerNetworkName(nwID string) (string, string, string, error) {
 	if err == nil {
 		return dnetOper.TenantName, dnetOper.NetworkName, dnetOper.ServiceName, nil
 	}
-	if pluginMode == master.SwarmMode {
+	if pluginMode == core.SwarmMode {
 		log.Errorf("Unable to find docknet info in objstore")
 		return "", "", "", err
 	}

diff --git a/netmaster/daemon/daemon.go b/netmaster/daemon/daemon.go
@@ -34,6 +34,7 @@ import (
 	"github.com/contiv/netplugin/netmaster/resources"
 	"github.com/contiv/netplugin/objdb"
 	"github.com/contiv/netplugin/utils"
+	"github.com/contiv/netplugin/utils/netutils"
 	"github.com/contiv/ofnet"
 	"github.com/gorilla/mux"
 
@@ -46,10 +47,14 @@ const leaderLockTTL = 30
 // MasterDaemon runs the daemon FSM
 type MasterDaemon struct {
 	// Public state
-	ListenURL    string // URL where netmaster listens for ext requests
-	ControlURL   string // URL where netmaster listens for ctrl pkts
-	ClusterStore string // state store URL
-	ClusterMode  string // cluster scheduler used docker/kubernetes/mesos etc
+	ListenURL          string // URL where netmaster listens for ext requests
+	ControlURL         string // URL where netmaster listens for ctrl pkts
+	ClusterStoreDriver string // state store driver name
+	ClusterStoreURL    string // state store endpoint
+	ClusterMode        string // cluster scheduler used docker/kubernetes/mesos etc
+	NetworkMode        string // network mode (vlan or vxlan)
+	NetForwardMode     string // forwarding mode (bridge or routing)
+	NetInfraType       string // infra type (aci or default)
 
 	// Private state
 	currState        string                          // Current state of the daemon
@@ -70,13 +75,13 @@ func (d *MasterDaemon) Init() {
 	// set cluster mode
 	err := master.SetClusterMode(d.ClusterMode)
 	if err != nil {
-		log.Fatalf("Failed to set cluster-mode. Error: %s", err)
+		log.Fatalf("Failed to set cluster-mode %q. Error: %s", d.ClusterMode, err)
 	}
 
 	// initialize state driver
-	d.stateDriver, err = initStateDriver(d.ClusterStore)
+	d.stateDriver, err = utils.NewStateDriver(d.ClusterStoreDriver, &core.InstanceInfo{DbURL: d.ClusterStoreURL})
 	if err != nil {
-		log.Fatalf("Failed to init state-store. Error: %s", err)
+		log.Fatalf("Failed to init state-store: driver %q, URLs %q. Error: %s", d.ClusterStoreDriver, d.ClusterStoreURL, err)
 	}
 
 	// Initialize resource manager
@@ -86,9 +91,9 @@ func (d *MasterDaemon) Init() {
 	}
 
 	// Create an objdb client
-	d.objdbClient, err = objdb.NewClient(d.ClusterStore)
+	d.objdbClient, err = objdb.InitClient(d.ClusterStoreDriver, []string{d.ClusterStoreURL})
 	if err != nil {
-		log.Fatalf("Error connecting to state store: %v. Err: %v", d.ClusterStore, err)
+		log.Fatalf("Error connecting to state store: driver %q, URLs %q. Err: %v", d.ClusterStoreDriver, d.ClusterStoreURL, err)
 	}
 }
 
@@ -404,7 +409,11 @@ func (d *MasterDaemon) runLeader() {
 	router := mux.NewRouter()
 
 	// Create a new api controller
-	d.apiController = objApi.NewAPIController(router, d.objdbClient, d.ClusterStore)
+	apiConfig := &objApi.APIControllerConfig{
+		NetForwardMode: d.NetForwardMode,
+		NetInfraType:   d.NetInfraType,
+	}
+	d.apiController = objApi.NewAPIController(router, d.objdbClient, apiConfig)
 
 	//Restore state from clusterStore
 	d.restoreCache()
@@ -447,6 +456,7 @@ func (d *MasterDaemon) startListeners(router *mux.Router, stopChan chan bool) {
 	server := &http.Server{Handler: router}
 	server.SetKeepAlivesEnabled(false)
 
+	// bind on external address
 	listener, err := net.Listen("tcp", d.ListenURL)
 	if nil != err {
 		log.Fatalln(err)
@@ -457,20 +467,25 @@ func (d *MasterDaemon) startListeners(router *mux.Router, stopChan chan bool) {
 
 	go server.Serve(listener)
 
-	listenURL := strings.Split(d.ListenURL, ":")
-	controlURL := strings.Split(d.ControlURL, ":")
+	if d.ControlURL != d.ListenURL {
+		externalAddr := strings.Split(d.ListenURL, ":")
+		internalAddr := strings.Split(d.ControlURL, ":")
+		if externalAddr[0] == "0.0.0.0" && externalAddr[1] == internalAddr[1] {
+			// ignore internal bind if external and internal are on the same port and external bind on 0.0.0.0
+			log.Infof("Ignore creating API listener on %q because %q covers it", d.ControlURL, d.ListenURL)
+		} else {
+			// it should fail-fast if ControlURL and ListenURL have other overlapping
+			ctrlListener, err := net.Listen("tcp", d.ControlURL)
+			if nil != err {
+				log.Fatalln(err)
+			}
+			log.Infof("Netmaster listening on %s for control packets", d.ControlURL)
+			ctrlListener = utils.ListenWrapper(ctrlListener)
+			defer ctrlListener.Close()
 
-	if (strings.Compare(listenURL[1], controlURL[1]) != 0) || (len(listenURL[0]) != 0 && strings.Compare(listenURL[0], "0.0.0.0") != 0 && strings.Compare(listenURL[0], controlURL[0]) != 0) {
-		ctrlListener, err := net.Listen("tcp", d.ControlURL)
-		if nil != err {
-			log.Fatalln(err)
+			// start server
+			go server.Serve(ctrlListener)
 		}
-		log.Infof("Netmaster listening on %s for control packets", d.ControlURL)
-		ctrlListener = utils.ListenWrapper(ctrlListener)
-		defer ctrlListener.Close()
-
-		// start server
-		go server.Serve(ctrlListener)
 	}
 
 	// Wait till we are asked to stop
@@ -584,7 +599,7 @@ func (d *MasterDaemon) getMasterInfo() (map[string]interface{}, error) {
 	info := make(map[string]interface{})
 
 	// get local ip
-	localIP, err := GetLocalAddr()
+	localIP, err := netutils.GetDefaultAddr()
 	if err != nil {
 		return nil, errors.New("error getting local IP address")
 	}

diff --git a/netmaster/daemon/utils.go b/netmaster/daemon/utils.go
@@ -21,7 +21,6 @@ import (
 	"net/http"
 	"net/http/httputil"
 	"net/url"
-	"strings"
 
 	"github.com/contiv/netplugin/core"
 	"github.com/contiv/netplugin/netmaster/mastercfg"
@@ -48,48 +47,11 @@ func getVersion(w http.ResponseWriter, r *http.Request) {
 	return
 }
 
-// initStateDriver creates a state driver based on the cluster store URL
-func initStateDriver(clusterStore string) (core.StateDriver, error) {
-	// parse the state store URL
-	parts := strings.Split(clusterStore, "://")
-	if len(parts) < 2 {
-		return nil, core.Errorf("Invalid state-store URL %q", clusterStore)
-	}
-	stateStore := parts[0]
-
-	// Make sure we support the statestore type
-	switch stateStore {
-	case utils.EtcdNameStr:
-	case utils.ConsulNameStr:
-	default:
-		return nil, core.Errorf("Unsupported state-store %q", stateStore)
-	}
-
-	// Setup instance info
-	instInfo := core.InstanceInfo{
-		DbURL: clusterStore,
-	}
-
-	return utils.NewStateDriver(stateStore, &instInfo)
-}
-
-// GetLocalAddr gets local address to be used
-func GetLocalAddr() (string, error) {
-	// get the ip address by local hostname
-	localIP, err := netutils.GetMyAddr()
-	if err == nil && netutils.IsAddrLocal(localIP) {
-		return localIP, nil
-	}
-
-	// Return first available address if we could not find by hostname
-	return netutils.GetFirstLocalAddr()
-}
-
 // slaveProxyHandler redirects to current master
 func slaveProxyHandler(w http.ResponseWriter, r *http.Request) {
 	log.Infof("proxy handler for %q ", r.URL.Path)
 
-	localIP, err := GetLocalAddr()
+	localIP, err := netutils.GetDefaultAddr()
 	if err != nil {
 		log.Fatalf("Error getting local IP address. Err: %v", err)
 	}