From 02fc4ffa934d6f883a9bf3415a131a1dbcae06e6 Mon Sep 17 00:00:00 2001 From: Arjun Baindur Date: Mon, 14 Nov 2022 15:13:14 -0800 Subject: [PATCH] Minor nodeletctl code reorg --- nodeletctl/pkg/nodeletctl/config.go | 127 ++++++++ nodeletctl/pkg/nodeletctl/nodeletctl.go | 396 +++++++++--------------- 2 files changed, 266 insertions(+), 257 deletions(-) create mode 100644 nodeletctl/pkg/nodeletctl/config.go diff --git a/nodeletctl/pkg/nodeletctl/config.go b/nodeletctl/pkg/nodeletctl/config.go new file mode 100644 index 00000000..3060146c --- /dev/null +++ b/nodeletctl/pkg/nodeletctl/config.go @@ -0,0 +1,127 @@ +package nodeletctl + +import ( + "fmt" + "os" + "path/filepath" + "text/template" + + "go.uber.org/zap" +) + +type NodeletConfig struct { + AllowWorkloadsOnMaster bool + CalicoV4Interface string + CalicoV6Interface string + ClusterId string + ContainerRuntime ContainerRuntimeConfig + EtcdClusterState string + HostId string + HostIp string + K8sApiPort string + MasterList *map[string]string + MasterIp string + MasterVipEnabled bool + MasterVipInterface string + MasterVipVrouterId int + Mtu string + Privileged string + NodeletRole string + UserImages []string + CoreDNSHostsFile string + IPv6Enabled bool + UseHostname bool + CalicoIP4 string + CalicoIP6 string + CalicoV4BlockSize int + CalicoV6BlockSize int + CalicoV6ContainersCidr string + CalicoV4ContainersCidr string + CalicoV4NATOutgoing bool + CalicoV6NATOutgoing bool + CalicoV4IpIpMode string + ContainersCidr string + ServicesCidr string +} + +func setNodeletClusterCfg(cfg *BootstrapConfig, nodelet *NodeletConfig) { + nodelet.AllowWorkloadsOnMaster = cfg.AllowWorkloadsOnMaster + nodelet.ClusterId = cfg.ClusterId + nodelet.ContainerRuntime = cfg.ContainerRuntime + nodelet.K8sApiPort = cfg.K8sApiPort + nodelet.MasterIp = cfg.MasterIp + nodelet.MasterVipEnabled = cfg.MasterVipEnabled + nodelet.MasterVipInterface = cfg.MasterVipInterface + nodelet.MasterVipVrouterId = cfg.MasterVipVrouterId + nodelet.Mtu = cfg.MTU + nodelet.Privileged = cfg.Privileged + nodelet.UserImages = cfg.UserImages + nodelet.CoreDNSHostsFile = cfg.DNS.HostsFile + nodelet.IPv6Enabled = cfg.IPv6Enabled + + //Set default Calico opts first + nodelet.CalicoV4Interface = cfg.Calico.V4Interface + nodelet.CalicoV4BlockSize = cfg.Calico.V4BlockSize + nodelet.CalicoV4IpIpMode = cfg.Calico.V4IpIpMode + nodelet.CalicoV4NATOutgoing = cfg.Calico.V4NATOutgoing + nodelet.ContainersCidr = cfg.Calico.V4ContainersCidr + nodelet.CalicoV6Interface = cfg.Calico.V6Interface + nodelet.CalicoV6BlockSize = cfg.Calico.V6BlockSize + nodelet.CalicoV6NATOutgoing = cfg.Calico.V6NATOutgoing + nodelet.CalicoV6ContainersCidr = cfg.Calico.V6ContainersCidr + + if cfg.IPv6Enabled { + // Always use hostname as node identifier for IPv6 + nodelet.UseHostname = true + // Disable IPv4 as dualstack not yet supported + nodelet.CalicoIP4 = "none" + nodelet.CalicoIP6 = "autodetect" + + // Need to set this field for v6, as it is used to set kube-proxy arg + nodelet.ContainersCidr = cfg.Calico.V6ContainersCidr + if cfg.ServicesCidr == "" { + nodelet.ServicesCidr = DefaultV6ServicesCidr + cfg.ServicesCidr = DefaultV6ServicesCidr + } else { + nodelet.ServicesCidr = cfg.ServicesCidr + } + } else { + // IPv4 only + nodelet.UseHostname = cfg.UseHostname + nodelet.CalicoIP4 = "autodetect" + nodelet.CalicoIP6 = "none" + if cfg.ServicesCidr == "" { + nodelet.ServicesCidr = DefaultV4ServicesCidr + cfg.ServicesCidr = DefaultV4ServicesCidr + } else { + nodelet.ServicesCidr = cfg.ServicesCidr + } + } +} + +func GenNodeletConfigLocal(host *NodeletConfig, templateName string) (string, error) { + nodeStateDir := filepath.Join(ClusterStateDir, host.ClusterId, host.HostId) + if _, err := os.Stat(nodeStateDir); os.IsNotExist(err) { + zap.S().Infof("Creating node state dir: %s\n", nodeStateDir) + if err := os.MkdirAll(nodeStateDir, 0777); err != nil { + return "", fmt.Errorf("Failed to create node state dir for host %s: %s", host.HostId, err) + } + } + + nodeletCfgFile := filepath.Join(nodeStateDir, NodeletConfigFile) + + t := template.Must(template.New(host.HostId).Parse(templateName)) + + fd, err := os.Create(nodeletCfgFile) + if err != nil { + return "", fmt.Errorf("Failed to Create nodelet config File: %s err: %s", nodeletCfgFile, err) + } + defer fd.Close() + + err = t.Execute(fd, host) + if err != nil { + return "", fmt.Errorf("template.Execute failed for file: %s err: %s\n", nodeletCfgFile, err) + } + + return nodeletCfgFile, nil +} diff --git a/nodeletctl/pkg/nodeletctl/nodeletctl.go b/nodeletctl/pkg/nodeletctl/nodeletctl.go index ed5f6e18..d6282722 100644 --- a/nodeletctl/pkg/nodeletctl/nodeletctl.go +++ b/nodeletctl/pkg/nodeletctl/nodeletctl.go @@ -7,7 +7,6 @@ import ( "os" "path/filepath" "sync" - "text/template" "time" "github.com/ghodss/yaml" @@ -69,41 +68,6 @@ type HostConfig struct { V6InterfaceOverride *string `json:"calicoV6Interface,omitempty"` } -type NodeletConfig struct { - AllowWorkloadsOnMaster bool - CalicoV4Interface string - CalicoV6Interface string - ClusterId string - ContainerRuntime ContainerRuntimeConfig - EtcdClusterState string - HostId string - HostIp string - K8sApiPort string - MasterList *map[string]string - MasterIp string - MasterVipEnabled bool - MasterVipInterface string - MasterVipVrouterId int - Mtu string - Privileged string - NodeletRole string - UserImages []string - CoreDNSHostsFile string - IPv6Enabled bool - UseHostname bool - CalicoIP4 string - CalicoIP6 string - CalicoV4BlockSize int - CalicoV6BlockSize int - CalicoV6ContainersCidr string - CalicoV4ContainersCidr string - CalicoV4NATOutgoing bool - CalicoV6NATOutgoing bool - CalicoV4IpIpMode string - ContainersCidr string - ServicesCidr string -} - type ClusterStatus struct { statusMap map[string]*NodeStatus } @@ -116,6 +80,68 @@ type NodeStatus struct { var globalClusterStatus *ClusterStatus +func InitBootstrapConfig() *BootstrapConfig { + calicoConfig := CalicoConfig{} + calicoConfig.V4BlockSize = 26 + calicoConfig.V6BlockSize = 122 + calicoConfig.V4ContainersCidr = DefaultCalicoV4Cidr + calicoConfig.V6ContainersCidr = DefaultCalicoV6Cidr + calicoConfig.V4NATOutgoing = true + calicoConfig.V6NATOutgoing = false + calicoConfig.V4IpIpMode = "Always" + calicoConfig.V4Interface = "first-found" + calicoConfig.V6Interface = "first-found" + + bootstrapCfg := &BootstrapConfig{ + AllowWorkloadsOnMaster: false, + ClusterId: DefaultClusterName, + ContainerRuntime: ContainerRuntimeConfig{"containerd", "systemd"}, + SSHUser: "root", + SSHPrivateKeyFile: "/root/.ssh/id_rsa", + Pf9KubePkg: NodeletTarSrc, + Privileged: "true", + K8sApiPort: "443", + MasterVipEnabled: false, + MTU: "1440", + IPv6Enabled: false, + UseHostname: false, + Calico: calicoConfig, + } + + return bootstrapCfg +} + +func ParseBootstrapConfig(cfgPath string) (*BootstrapConfig, error) { + zap.S().Infof("ParseBootstrapConfig cfgPath: %s", cfgPath) + cfgFile, err := ioutil.ReadFile(cfgPath) + if err != nil { + return nil, fmt.Errorf("Error opening bootstrap config file: %s", cfgFile) + } + + bootstrapConfig := InitBootstrapConfig() + err = yaml.Unmarshal(cfgFile, bootstrapConfig) + if err != nil { + return nil, fmt.Errorf("error decoding bootstrap config: %v", err) + } + + if err := isClusterCfgValid(bootstrapConfig); err != nil { + return nil, fmt.Errorf("Invalid cluster config: %s", err) + } + + if len(bootstrapConfig.DNS.InlineHosts) > 0 { + // If custom hosts are specified, save to /etc/pf9/hosts and upload to each node to use core CoreDNS + bootstrapConfig.DNS.HostsFile, err = WriteHostsFileForEntries(bootstrapConfig.ClusterId, bootstrapConfig.DNS.InlineHosts) + if err != nil { + return nil, fmt.Errorf("Failed to generate custom hosts file: %s", err) + } + } else if bootstrapConfig.DNS.HostsFile == "" { + // If custom hosts and custom file are both empty, use local /etc/hosts as default + bootstrapConfig.DNS.HostsFile = "/etc/hosts" + } + + return bootstrapConfig, nil +} + func CreateCluster(cfgPath string) error { clusterCfg, err := ParseBootstrapConfig(cfgPath) if err != nil { @@ -149,6 +175,83 @@ func CreateCluster(cfgPath string) error { return nil } +func DeployCluster(clusterCfg *BootstrapConfig) error { + zap.S().Infof("Deploying cluster %s", clusterCfg.ClusterId) + if clusterCfg.CertsDir == "" { + certsDir, err := GenCALocal(clusterCfg.ClusterId) + if err != nil { + return fmt.Errorf("Cert generation failed: %s\n", err) + } + clusterCfg.CertsDir = certsDir + } + + err := trustCA(clusterCfg.CertsDir) + if err != nil { + zap.S().Errorf("error adding nodelet Root CA as trusted certs: %s\n", err) + return fmt.Errorf("error adding nodelet Root CA as trusted certs: %s\n", err) + } + + if err := GenKubeconfig(clusterCfg); err != nil { + zap.S().Infof("Failed to generate kubeconfig: %s\n", err) + return err + } + + globalClusterStatus = new(ClusterStatus) + globalClusterStatus.statusMap = make(map[string]*NodeStatus) + var masterList = make(map[string]string) + for _, host := range clusterCfg.MasterNodes { + if host.NodeIP != nil { + masterList[host.NodeName] = *host.NodeIP + } else { + masterList[host.NodeName] = host.NodeName + } + } + + for numMaster, host := range clusterCfg.MasterNodes { + zap.S().Infof("Deploying master node %s", host.NodeName) + nodeletCfg := new(NodeletConfig) + setNodeletClusterCfg(clusterCfg, nodeletCfg) + nodeletCfg.HostId = host.NodeName + if host.NodeIP != nil { + nodeletCfg.HostIp = *host.NodeIP + } else { + nodeletCfg.HostIp = host.NodeName + } + nodeletCfg.NodeletRole = "master" + nodeletCfg.MasterList = &masterList + nodeletCfg.EtcdClusterState = "new" + + nodeletSrcFile, err := GenNodeletConfigLocal(nodeletCfg, masterNodeletConfigTmpl) + if err != nil { + zap.S().Infof("Failed to generate config: %s", err) + return fmt.Errorf("Failed to generate config: %s", err) + } + zap.S().Debugf("master nodeletsrc file %s", nodeletSrcFile) + deployer, err := GetNodeletDeployer(clusterCfg, globalClusterStatus, nodeletCfg, nodeletSrcFile) + if err != nil { + zap.S().Errorf("failed to get nodelet deployer: %v", err) + return fmt.Errorf("failed to get nodelet deployer: %v", err) + } + globalClusterStatus.statusMap[host.NodeName] = &NodeStatus{ + deployer: deployer, + } + + converged, err := deployer.SpawnMaster(numMaster) + zap.S().Infof("Master status: %s\n", converged) + if err != nil { + zap.S().Infof("err = %s\n", err) + } + } + + if err := DeployWorkers(clusterCfg, globalClusterStatus, &clusterCfg.WorkerNodes); err != nil { + return fmt.Errorf("ScaleCluster failed to deploy new workers: %s", err) + } + + SyncNodes(clusterCfg, nil) + zap.S().Infof("Cluster deployed successfully") + return nil +} + func UpgradeCluster(cfgPath string) error { clusterCfg, err := ParseBootstrapConfig(cfgPath) @@ -284,145 +387,6 @@ func UpgradeWorkers(clusterCfg *BootstrapConfig, clusterStatus *ClusterStatus) e return nil } -func InitBootstrapConfig() *BootstrapConfig { - calicoConfig := CalicoConfig{} - calicoConfig.V4BlockSize = 26 - calicoConfig.V6BlockSize = 122 - calicoConfig.V4ContainersCidr = DefaultCalicoV4Cidr - calicoConfig.V6ContainersCidr = DefaultCalicoV6Cidr - calicoConfig.V4NATOutgoing = true - calicoConfig.V6NATOutgoing = false - calicoConfig.V4IpIpMode = "Always" - calicoConfig.V4Interface = "first-found" - calicoConfig.V6Interface = "first-found" - - bootstrapCfg := &BootstrapConfig{ - AllowWorkloadsOnMaster: false, - ClusterId: DefaultClusterName, - ContainerRuntime: ContainerRuntimeConfig{"containerd", "systemd"}, - SSHUser: "root", - SSHPrivateKeyFile: "/root/.ssh/id_rsa", - Pf9KubePkg: NodeletTarSrc, - Privileged: "true", - K8sApiPort: "443", - MasterVipEnabled: false, - MTU: "1440", - IPv6Enabled: false, - UseHostname: false, - Calico: calicoConfig, - } - - return bootstrapCfg -} - -func ParseBootstrapConfig(cfgPath string) (*BootstrapConfig, error) { - zap.S().Infof("ParseBootstrapConfig cfgPath: %s", cfgPath) - cfgFile, err := ioutil.ReadFile(cfgPath) - if err != nil { - return nil, fmt.Errorf("Error opening bootstrap config file: %s", cfgFile) - } - - bootstrapConfig := InitBootstrapConfig() - err = yaml.Unmarshal(cfgFile, bootstrapConfig) - if err != nil { - return nil, fmt.Errorf("error decoding bootstrap config: %v", err) - } - - if err := isClusterCfgValid(bootstrapConfig); err != nil { - return nil, fmt.Errorf("Invalid cluster config: %s", err) - } - - if len(bootstrapConfig.DNS.InlineHosts) > 0 { - // If custom hosts are specified, save to /etc/pf9/hosts and upload to each node to use core CoreDNS - bootstrapConfig.DNS.HostsFile, err = WriteHostsFileForEntries(bootstrapConfig.ClusterId, bootstrapConfig.DNS.InlineHosts) - if err != nil { - return nil, fmt.Errorf("Failed to generate custom hosts file: %s", err) - } - } else if bootstrapConfig.DNS.HostsFile == "" { - // If custom hosts and custom file are both empty, use local /etc/hosts as default - bootstrapConfig.DNS.HostsFile = "/etc/hosts" - } - - return bootstrapConfig, nil -} - -func DeployCluster(clusterCfg *BootstrapConfig) error { - zap.S().Infof("Deploying cluster %s", clusterCfg.ClusterId) - if clusterCfg.CertsDir == "" { - certsDir, err := GenCALocal(clusterCfg.ClusterId) - if err != nil { - return fmt.Errorf("Cert generation failed: %s\n", err) - } - clusterCfg.CertsDir = certsDir - } - - err := trustCA(clusterCfg.CertsDir) - if err != nil { - zap.S().Errorf("error adding nodelet Root CA as trusted certs: %s\n", err) - return fmt.Errorf("error adding nodelet Root CA as trusted certs: %s\n", err) - } - - if err := GenKubeconfig(clusterCfg); err != nil { - zap.S().Infof("Failed to generate kubeconfig: %s\n", err) - return err - } - - globalClusterStatus = new(ClusterStatus) - globalClusterStatus.statusMap = make(map[string]*NodeStatus) - var masterList = make(map[string]string) - for _, host := range clusterCfg.MasterNodes { - if host.NodeIP != nil { - masterList[host.NodeName] = *host.NodeIP - } else { - masterList[host.NodeName] = host.NodeName - } - } - - for numMaster, host := range clusterCfg.MasterNodes { - zap.S().Infof("Deploying master node %s", host.NodeName) - nodeletCfg := new(NodeletConfig) - setNodeletClusterCfg(clusterCfg, nodeletCfg) - nodeletCfg.HostId = host.NodeName - if host.NodeIP != nil { - nodeletCfg.HostIp = *host.NodeIP - } else { - nodeletCfg.HostIp = host.NodeName - } - nodeletCfg.NodeletRole = "master" - nodeletCfg.MasterList = &masterList - nodeletCfg.EtcdClusterState = "new" - - nodeletSrcFile, err := GenNodeletConfigLocal(nodeletCfg, masterNodeletConfigTmpl) - if err != nil { - zap.S().Infof("Failed to generate config: %s", err) - return fmt.Errorf("Failed to generate config: %s", err) - } - zap.S().Debugf("master nodeletsrc file %s", nodeletSrcFile) - deployer, err := GetNodeletDeployer(clusterCfg, globalClusterStatus, nodeletCfg, nodeletSrcFile) - if err != nil { - zap.S().Errorf("failed to get nodelet deployer: %v", err) - return fmt.Errorf("failed to get nodelet deployer: %v", err) - } - globalClusterStatus.statusMap[host.NodeName] = &NodeStatus{ - deployer: deployer, - } - - converged, err := deployer.SpawnMaster(numMaster) - zap.S().Infof("Master status: %s\n", converged) - if err != nil { - zap.S().Infof("err = %s\n", err) - } - } - - if err := DeployWorkers(clusterCfg, globalClusterStatus, &clusterCfg.WorkerNodes); err != nil { - return fmt.Errorf("ScaleCluster failed to deploy new workers: %s", err) - } - - SyncNodes(clusterCfg, nil) - zap.S().Infof("Cluster deployed successfully") - return nil -} - func SetClusterNodeStatus(status *ClusterStatus, nodeName, health string, err error) { status.statusMap[nodeName].nodeHealth = health status.statusMap[nodeName].errMsg = err @@ -453,88 +417,6 @@ func SyncAndRetry(clusterCfg *BootstrapConfig, nodeletStatus *ClusterStatus, nod close(done) } -func setNodeletClusterCfg(cfg *BootstrapConfig, nodelet *NodeletConfig) { - nodelet.AllowWorkloadsOnMaster = cfg.AllowWorkloadsOnMaster - nodelet.ClusterId = cfg.ClusterId - nodelet.ContainerRuntime = cfg.ContainerRuntime - nodelet.K8sApiPort = cfg.K8sApiPort - nodelet.MasterIp = cfg.MasterIp - nodelet.MasterVipEnabled = cfg.MasterVipEnabled - nodelet.MasterVipInterface = cfg.MasterVipInterface - nodelet.MasterVipVrouterId = cfg.MasterVipVrouterId - nodelet.Mtu = cfg.MTU - nodelet.Privileged = cfg.Privileged - nodelet.UserImages = cfg.UserImages - nodelet.CoreDNSHostsFile = cfg.DNS.HostsFile - nodelet.IPv6Enabled = cfg.IPv6Enabled - - //Set default Calico opts first - nodelet.CalicoV4Interface = cfg.Calico.V4Interface - nodelet.CalicoV4BlockSize = cfg.Calico.V4BlockSize - nodelet.CalicoV4IpIpMode = cfg.Calico.V4IpIpMode - nodelet.CalicoV4NATOutgoing = cfg.Calico.V4NATOutgoing - nodelet.ContainersCidr = cfg.Calico.V4ContainersCidr - nodelet.CalicoV6Interface = cfg.Calico.V6Interface - nodelet.CalicoV6BlockSize = cfg.Calico.V6BlockSize - nodelet.CalicoV6NATOutgoing = cfg.Calico.V6NATOutgoing - nodelet.CalicoV6ContainersCidr = cfg.Calico.V6ContainersCidr - - if cfg.IPv6Enabled { - // Always use hostname as node identifier for IPv6 - nodelet.UseHostname = true - // Disable IPv4 as dualstack not yet supported - nodelet.CalicoIP4 = "none" - nodelet.CalicoIP6 = "autodetect" - - // Need to set this field for v6, as it is used to set kube-proxy arg - nodelet.ContainersCidr = cfg.Calico.V6ContainersCidr - if cfg.ServicesCidr == "" { - nodelet.ServicesCidr = DefaultV6ServicesCidr - cfg.ServicesCidr = DefaultV6ServicesCidr - } else { - nodelet.ServicesCidr = cfg.ServicesCidr - } - } else { - // IPv4 only - nodelet.UseHostname = cfg.UseHostname - nodelet.CalicoIP4 = "autodetect" - nodelet.CalicoIP6 = "none" - if cfg.ServicesCidr == "" { - nodelet.ServicesCidr = DefaultV4ServicesCidr - cfg.ServicesCidr = DefaultV4ServicesCidr - } else { - nodelet.ServicesCidr = cfg.ServicesCidr - } - } -} - -func GenNodeletConfigLocal(host *NodeletConfig, templateName string) (string, error) { - nodeStateDir := filepath.Join(ClusterStateDir, host.ClusterId, host.HostId) - if _, err := os.Stat(nodeStateDir); os.IsNotExist(err) { - zap.S().Infof("Creating node state dir: %s\n", nodeStateDir) - if err := os.MkdirAll(nodeStateDir, 0777); err != nil { - return "", fmt.Errorf("Failed to create node state dir for host %s: %s", host.HostId, err) - } - } - - nodeletCfgFile := filepath.Join(nodeStateDir, NodeletConfigFile) - - t := template.Must(template.New(host.HostId).Parse(templateName)) - - fd, err := os.Create(nodeletCfgFile) - if err != nil { - return "", fmt.Errorf("Failed to Create nodelet config File: %s err: %s", nodeletCfgFile, err) - } - defer fd.Close() - - err = t.Execute(fd, host) - if err != nil { - return "", fmt.Errorf("template.Execute failed for file: %s err: %s\n", nodeletCfgFile, err) - } - - return nodeletCfgFile, nil -} - func DeleteCluster(cfgPath string) error { clusterCfg, err := ParseBootstrapConfig(cfgPath) if err != nil {